Embedded compiler

CMakeLists.txt

@@ -90,6 +90,14 @@ endif ()
 
 set (COMPILER_FLAGS                      "${COMPILER_FLAGS} ${CXX11_ABI_FLAGS}")
 
+find_program (LINKER_PROGRAM NAMES lld gold ld)
+get_filename_component (LINKER_NAME ${LINKER_PROGRAM} NAME CACHE)
+
+set (CMAKE_EXE_LINKER_FLAGS              "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
+if (LINKER_NAME STREQUAL lld)
+    set (CMAKE_EXE_LINKER_FLAGS          "${CMAKE_EXE_LINKER_FLAGS} -L/usr/local/lib")
+endif ()
+
 option (PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
 if (PIPE)
     set (COMPILER_FLAGS                  "${COMPILER_FLAGS} -pipe")
@@ -159,12 +167,6 @@ endif ()
 set (SAN_FLAGS "-O3 -g -fno-omit-frame-pointer")
 set (CMAKE_CXX_FLAGS_ASAN                "${CMAKE_CXX_FLAGS_ASAN}  ${SAN_FLAGS} -fsanitize=address")
 set (CMAKE_C_FLAGS_ASAN                  "${CMAKE_C_FLAGS_ASAN}    ${SAN_FLAGS} -fsanitize=address")
-if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    # -fuse-ld=gold - fix linkage for gcc-5.4, gcc-6.1
-    # See more in http://stackoverflow.com/questions/37603238/fsanitize-not-using-gold-linker-in-gcc-6-1
-    set (CMAKE_CXX_FLAGS_ASAN            "${CMAKE_CXX_FLAGS_ASAN} -fuse-ld=gold")
-    set (CMAKE_C_FLAGS_ASAN              "${CMAKE_C_FLAGS_ASAN}   -fuse-ld=gold")
-endif ()
 set (CMAKE_CXX_FLAGS_UBSAN               "${CMAKE_CXX_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined")
 set (CMAKE_C_FLAGS_UBSAN                 "${CMAKE_C_FLAGS_UBSAN}   ${SAN_FLAGS} -fsanitize=undefined")
 set (CMAKE_CXX_FLAGS_MSAN                "${CMAKE_CXX_FLAGS_MSAN}  ${SAN_FLAGS} -fsanitize=memory")
@@ -236,6 +238,7 @@ include (cmake/find_zookeeper.cmake)
 include (cmake/find_re2.cmake)
 include (cmake/find_rdkafka.cmake)
 include (cmake/find_capnp.cmake)
+include (cmake/find_llvm.cmake)
 
 include (cmake/find_contrib_lib.cmake)
 find_contrib_lib(cityhash)

cmake/find_llvm.cmake

@@ -0,0 +1,87 @@
+option (USE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" FALSE)
+
+if (USE_EMBEDDED_COMPILER)
+    # Based on source code of YT.
+    # Authors: Ivan Puzyrevskiy, Alexey Lukyanchikov, Ruslan Savchenko.
+
+    #  Find LLVM includes and libraries.
+    #
+    #  LLVM_VERSION      - LLVM version.
+    #  LLVM_INCLUDE_DIRS - Directory containing LLVM headers.
+    #  LLVM_LIBRARY_DIRS - Directory containing LLVM libraries.
+    #  LLVM_CXXFLAGS     - C++ compiler flags for files that include LLVM headers.
+    #  LLVM_FOUND        - True if LLVM was found.
+
+    #  llvm_map_components_to_libraries - Maps LLVM used components to required libraries.
+    #  Usage: llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES core jit interpreter native ...)
+
+    find_program(LLVM_CONFIG_EXECUTABLE
+        NAMES llvm-config
+        PATHS $ENV{LLVM_ROOT}/bin)
+
+    mark_as_advanced(LLVM_CONFIG_EXECUTABLE)
+
+    if(NOT LLVM_CONFIG_EXECUTABLE)
+        message(FATAL_ERROR "Cannot find LLVM (looking for `llvm-config`). Please, provide LLVM_ROOT environment variable.")
+    else()
+        set(LLVM_FOUND TRUE)
+
+        execute_process(
+            COMMAND ${LLVM_CONFIG_EXECUTABLE} --version
+            OUTPUT_VARIABLE LLVM_VERSION
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+        if(LLVM_VERSION VERSION_LESS "5")
+            message(FATAL_ERROR "LLVM 5+ is required.")
+        endif()
+
+        execute_process(
+            COMMAND ${LLVM_CONFIG_EXECUTABLE} --includedir
+            OUTPUT_VARIABLE LLVM_INCLUDE_DIRS
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+        execute_process(
+            COMMAND ${LLVM_CONFIG_EXECUTABLE} --libdir
+            OUTPUT_VARIABLE LLVM_LIBRARY_DIRS
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+        execute_process(
+            COMMAND ${LLVM_CONFIG_EXECUTABLE} --cxxflags
+            OUTPUT_VARIABLE LLVM_CXXFLAGS
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+        execute_process(
+            COMMAND ${LLVM_CONFIG_EXECUTABLE} --targets-built
+            OUTPUT_VARIABLE LLVM_TARGETS_BUILT
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+        string(REPLACE " " ";" LLVM_TARGETS_BUILT "${LLVM_TARGETS_BUILT}")
+
+        # Get the link libs we need.
+        function(llvm_map_components_to_libraries RESULT)
+            execute_process(
+                COMMAND ${LLVM_CONFIG_EXECUTABLE} --libs ${ARGN}
+                OUTPUT_VARIABLE _tmp
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+            string(REPLACE " " ";" _libs_module "${_tmp}")
+
+            message(STATUS "LLVM Libraries for '${ARGN}': ${_libs_module}")
+
+            execute_process(
+                COMMAND ${LLVM_CONFIG_EXECUTABLE} --system-libs ${ARGN}
+                OUTPUT_VARIABLE _libs_system
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+            string(REPLACE "\n" " " _libs_system "${_libs_system}")
+            string(REPLACE "    " " " _libs_system "${_libs_system}")
+            string(REPLACE " "    ";" _libs_system "${_libs_system}")
+
+            set(${RESULT} ${_libs_module} ${_libs_system} PARENT_SCOPE)
+        endfunction(llvm_map_components_to_libraries)
+
+        message(STATUS "LLVM Include Directory: ${LLVM_INCLUDE_DIRS}")
+        message(STATUS "LLVM Library Directory: ${LLVM_LIBRARY_DIRS}")
+        message(STATUS "LLVM C++ Compiler: ${LLVM_CXXFLAGS}")
+    endif()
+endif()

dbms/src/Common/config.h.in

@@ -8,6 +8,7 @@
 #cmakedefine01 USE_VECTORCLASS
 #cmakedefine01 USE_RDKAFKA
 #cmakedefine01 USE_CAPNP
+#cmakedefine01 USE_EMBEDDED_COMPILER
 #cmakedefine01 Poco_DataODBC_FOUND
 #cmakedefine01 Poco_MongoDB_FOUND
 #cmakedefine01 Poco_NetSSL_FOUND

dbms/src/Interpreters/CMakeLists.txt

@@ -5,17 +5,14 @@ else ()
     set (PATH_SHARE "/usr/share" CACHE STRING "")
 endif ()
 
-if (INTERNAL_COMPILER_SAME)
-    set (INTERNAL_COMPILER_EXECUTABLE "${CMAKE_CXX_COMPILER}" CACHE STRING "")
-else ()
-    set (INTERNAL_COMPILER_EXECUTABLE "${PATH_SHARE}/clickhouse/bin/clang" CACHE STRING "")
+if (USE_EMBEDDED_COMPILER)
+    set (INTERNAL_COMPILER_EXECUTABLE "clickhouse-clang" CACHE STRING "")
+    set (INTERNAL_LINKER_EXECUTABLE "clickhouse-lld" CACHE STRING "")
 endif ()
 
 set (INTERNAL_COMPILER_NO_WARNING OFF CACHE BOOL "")
 
-#original string:
-# -x c++ -std=gnu++1y -O3 -g -Wall -Werror -Wnon-virtual-dtor -march=native -msse4 -mpopcnt -D NDEBUG -D_GLIBCXX_USE_CXX11_ABI=0 -shared -fPIC -fvisibility=hidden -fno-implement-inlines
-set (INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -shared -fPIC -fvisibility=hidden -fno-implement-inlines " CACHE STRING "")
+set (INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines " CACHE STRING "")
 string(REPLACE "-no-pie" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
 if (INTERNAL_COMPILER_NO_WARNING)
     string (REPLACE "-Wall" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})

dbms/src/Interpreters/Compiler.cpp

@@ -203,6 +203,7 @@ void Compiler::compile(
 
     std::string prefix = path + "/" + file_name;
     std::string cpp_file_path = prefix + ".cpp";
+    std::string o_file_path = prefix + ".o";
     std::string so_file_path = prefix + ".so";
     std::string so_tmp_file_path = prefix + ".so.tmp";
 
@@ -217,29 +218,35 @@ void Compiler::compile(
 
     /// Slightly unconvenient.
     command <<
-        "LD_LIBRARY_PATH=" PATH_SHARE "/clickhouse/bin/"
-        " " INTERNAL_COMPILER_EXECUTABLE
-        " -B " PATH_SHARE "/clickhouse/bin/"
-        " " INTERNAL_COMPILER_FLAGS
-#if INTERNAL_COMPILER_CUSTOM_ROOT
-        " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/"
-        " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/"
-        " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/"
-        " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/"
-        " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/c++/*/"
-        " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include/"
-        " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/clang/*/include/"
-#endif
-        " -I " INTERNAL_COMPILER_HEADERS "/dbms/src/"
-        " -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/"
-        " -I " INTERNAL_COMPILER_HEADERS "/contrib/libpcg-random/include/"
-        " -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
-        " -I " INTERNAL_Poco_Foundation_INCLUDE_DIR
-        " -I " INTERNAL_Boost_INCLUDE_DIRS
-        " -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/"
-        " " << additional_compiler_flags <<
-        " -o " << so_tmp_file_path << " " << cpp_file_path
-        << " 2>&1 || echo Exit code: $?";
+        "("
+            INTERNAL_COMPILER_EXECUTABLE
+            " " INTERNAL_COMPILER_FLAGS
+    #if INTERNAL_COMPILER_CUSTOM_ROOT
+            " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/"
+            " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/"
+            " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/"
+            " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/"
+            " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/c++/*/"
+            " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include/"
+            " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/clang/*/include/"
+    #endif
+            " -I " INTERNAL_COMPILER_HEADERS "/dbms/src/"
+            " -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/"
+            " -I " INTERNAL_COMPILER_HEADERS "/contrib/libpcg-random/include/"
+            " -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
+            " -I " INTERNAL_Poco_Foundation_INCLUDE_DIR
+            " -I " INTERNAL_Boost_INCLUDE_DIRS
+            " -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/"
+            " " << additional_compiler_flags <<
+            " -c -o " << o_file_path << " " << cpp_file_path
+            << " 2>&1"
+        ") && ("
+            INTERNAL_LINKER_EXECUTABLE
+            " -shared"
+            " -o " << so_tmp_file_path
+            << " " << o_file_path
+            << " 2>&1"
+        ") || echo Return code: $?";
 
     std::string compile_result;
 
@@ -254,6 +261,7 @@ void Compiler::compile(
 
     /// If there was an error before, the file with the code remains for viewing.
     Poco::File(cpp_file_path).remove();
+    Poco::File(o_file_path).remove();
 
     Poco::File(so_tmp_file_path).renameTo(so_file_path);
     SharedLibraryPtr lib(new SharedLibrary(so_file_path));

dbms/src/Interpreters/config_compile.h.in

@@ -3,6 +3,7 @@
 #cmakedefine PATH_SHARE "@PATH_SHARE@"
 #cmakedefine INTERNAL_COMPILER_FLAGS "@INTERNAL_COMPILER_FLAGS@"
 #cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@"
+#cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@"
 #cmakedefine INTERNAL_COMPILER_HEADERS "@INTERNAL_COMPILER_HEADERS@"
 #cmakedefine INTERNAL_COMPILER_HEADERS_ROOT "@INTERNAL_COMPILER_HEADERS_ROOT@"
 #cmakedefine01 INTERNAL_COMPILER_CUSTOM_ROOT

dbms/src/Server/CMakeLists.txt

@@ -43,7 +43,6 @@ target_link_libraries (clickhouse-compressor-lib dbms ${Boost_PROGRAM_OPTIONS_LI
 add_library (clickhouse-format Format.cpp)
 target_link_libraries (clickhouse-format dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
-
 add_executable(clickhouse main.cpp)
 target_include_directories(clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
 target_link_libraries(clickhouse
@@ -55,6 +54,12 @@ target_link_libraries(clickhouse
     clickhouse-extract-from-config
     clickhouse-compressor-lib
     clickhouse-format)
+
+if (USE_EMBEDDED_COMPILER)
+    add_subdirectory (Compiler)
+    target_link_libraries (clickhouse clickhouse-compiler)
+endif ()
+
 INSTALL(TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 # make symbolic links to concrete clickhouse applications
 macro(install_symlink_to_clickhouse app)
@@ -68,6 +73,11 @@ install_symlink_to_clickhouse(clickhouse-benchmark)
 install_symlink_to_clickhouse(clickhouse-performance-test)
 install_symlink_to_clickhouse(clickhouse-format)
 
+if (USE_EMBEDDED_COMPILER)
+    install_symlink_to_clickhouse(clickhouse-clang)
+    install_symlink_to_clickhouse(clickhouse-lld)
+endif ()
+
 INSTALL(
     FILES config.xml users.xml
     DESTINATION  ${CLICKHOUSE_ETC_DIR}/clickhouse-server

dbms/src/Server/Compiler/CMakeLists.txt

@@ -0,0 +1,40 @@
+add_library(clickhouse-compiler
+    driver.cpp
+    cc1_main.cpp
+    cc1as_main.cpp
+    lld.cpp)
+
+target_compile_options(clickhouse-compiler PRIVATE -fno-rtti)
+
+llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all)
+
+# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency.
+list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz")
+
+# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
+
+target_link_libraries(clickhouse-compiler
+
+clangBasic clangCodeGen clangDriver clangFrontend clangFrontendTool
+clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
+clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
+clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
+
+lldCOFF
+lldDriver
+lldELF
+lldMinGW
+lldMachO
+lldReaderWriter
+lldYAML
+lldCommon
+lldCore
+
+${REQUIRED_LLVM_LIBRARIES}
+
+Polly
+PollyISL
+PollyPPCG
+
+libtinfo.a
+common)

dbms/src/Server/Compiler/LICENSE.TXT

@@ -0,0 +1,63 @@
+==============================================================================
+LLVM Release License
+==============================================================================
+University of Illinois/NCSA
+Open Source License
+
+Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign.
+All rights reserved.
+
+Developed by:
+
+    LLVM Team
+
+    University of Illinois at Urbana-Champaign
+
+    http://llvm.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimers.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimers in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the names of the LLVM Team, University of Illinois at
+      Urbana-Champaign, nor the names of its contributors may be used to
+      endorse or promote products derived from this Software without specific
+      prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
+
+==============================================================================
+The LLVM software contains code written by third parties.  Such software will
+have its own individual LICENSE.TXT file in the directory in which it appears.
+This file will describe the copyrights, license, and restrictions which apply
+to that code.
+
+The disclaimer of warranty in the University of Illinois Open Source License
+applies to all code in the LLVM Distribution, and nothing in any of the
+other licenses gives permission to use the names of the LLVM Team or the
+University of Illinois to endorse or promote products derived from this
+Software.
+
+The following pieces of software have additional or alternate copyrights,
+licenses, and/or restrictions:
+
+Program             Directory
+-------             ---------
+<none yet>
+