From 9b18c6cb76247d7c198e0fd0d4e47fc2fde2cd7a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 12 Nov 2017 15:58:40 +0300 Subject: [PATCH] Embedded compiler (#1484) * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. * Embedded compiler: development [#CLICKHOUSE-3396]. --- CMakeLists.txt | 15 +- cmake/find_llvm.cmake | 87 +++ dbms/src/Common/config.h.in | 1 + dbms/src/Interpreters/CMakeLists.txt | 11 +- dbms/src/Interpreters/Compiler.cpp | 54 +- dbms/src/Interpreters/config_compile.h.in | 1 + dbms/src/Server/CMakeLists.txt | 12 +- dbms/src/Server/Compiler/CMakeLists.txt | 40 ++ dbms/src/Server/Compiler/LICENSE.TXT | 63 ++ dbms/src/Server/Compiler/cc1_main.cpp | 239 ++++++++ dbms/src/Server/Compiler/cc1as_main.cpp | 541 ++++++++++++++++++ dbms/src/Server/Compiler/driver.cpp | 520 +++++++++++++++++ dbms/src/Server/Compiler/lld.cpp | 40 ++ dbms/src/Server/main.cpp | 19 +- debian/copy_clang_binaries.sh | 30 - debian/rules | 7 +- libs/libglibc-compatibility/CMakeLists.txt | 13 +- .../glibc-compatibility.c | 5 + libs/libglibc-compatibility/musl/futimens.c | 38 ++ libs/libglibc-compatibility/musl/pipe2.c | 17 +- .../libglibc-compatibility/musl/posix_spawn.c | 107 ++++ .../musl/sched_cpucount.c | 11 + libs/libglibc-compatibility/musl/syscall.h | 9 + libs/libglibc-compatibility/musl/syscall.s | 13 + .../libglibc-compatibility/musl/syscall_ret.c | 11 + release | 6 +- 26 files changed, 1825 insertions(+), 85 deletions(-) create mode 100644 cmake/find_llvm.cmake create mode 100644 dbms/src/Server/Compiler/CMakeLists.txt create mode 100644 dbms/src/Server/Compiler/LICENSE.TXT create mode 100644 dbms/src/Server/Compiler/cc1_main.cpp create mode 100644 dbms/src/Server/Compiler/cc1as_main.cpp create mode 100644 dbms/src/Server/Compiler/driver.cpp create mode 100644 dbms/src/Server/Compiler/lld.cpp delete mode 100755 debian/copy_clang_binaries.sh create mode 100644 libs/libglibc-compatibility/musl/futimens.c create mode 100644 libs/libglibc-compatibility/musl/posix_spawn.c create mode 100644 libs/libglibc-compatibility/musl/sched_cpucount.c create mode 100644 libs/libglibc-compatibility/musl/syscall.h create mode 100644 libs/libglibc-compatibility/musl/syscall.s create mode 100644 libs/libglibc-compatibility/musl/syscall_ret.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 07e77e880280..a3ccdf5a2fb7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,14 @@ endif () set (COMPILER_FLAGS "${COMPILER_FLAGS} ${CXX11_ABI_FLAGS}") +find_program (LINKER_PROGRAM NAMES lld gold ld) +get_filename_component (LINKER_NAME ${LINKER_PROGRAM} NAME CACHE) + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") +if (LINKER_NAME STREQUAL lld) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L/usr/local/lib") +endif () + option (PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON) if (PIPE) set (COMPILER_FLAGS "${COMPILER_FLAGS} -pipe") @@ -159,12 +167,6 @@ endif () set (SAN_FLAGS "-O3 -g -fno-omit-frame-pointer") set (CMAKE_CXX_FLAGS_ASAN "${CMAKE_CXX_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address") set (CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address") -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # -fuse-ld=gold - fix linkage for gcc-5.4, gcc-6.1 - # See more in http://stackoverflow.com/questions/37603238/fsanitize-not-using-gold-linker-in-gcc-6-1 - set (CMAKE_CXX_FLAGS_ASAN "${CMAKE_CXX_FLAGS_ASAN} -fuse-ld=gold") - set (CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} -fuse-ld=gold") -endif () set (CMAKE_CXX_FLAGS_UBSAN "${CMAKE_CXX_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined") set (CMAKE_C_FLAGS_UBSAN "${CMAKE_C_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined") set (CMAKE_CXX_FLAGS_MSAN "${CMAKE_CXX_FLAGS_MSAN} ${SAN_FLAGS} -fsanitize=memory") @@ -236,6 +238,7 @@ include (cmake/find_zookeeper.cmake) include (cmake/find_re2.cmake) include (cmake/find_rdkafka.cmake) include (cmake/find_capnp.cmake) +include (cmake/find_llvm.cmake) include (cmake/find_contrib_lib.cmake) find_contrib_lib(cityhash) diff --git a/cmake/find_llvm.cmake b/cmake/find_llvm.cmake new file mode 100644 index 000000000000..fc7b1a63a18a --- /dev/null +++ b/cmake/find_llvm.cmake @@ -0,0 +1,87 @@ +option (USE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" FALSE) + +if (USE_EMBEDDED_COMPILER) + # Based on source code of YT. + # Authors: Ivan Puzyrevskiy, Alexey Lukyanchikov, Ruslan Savchenko. + + # Find LLVM includes and libraries. + # + # LLVM_VERSION - LLVM version. + # LLVM_INCLUDE_DIRS - Directory containing LLVM headers. + # LLVM_LIBRARY_DIRS - Directory containing LLVM libraries. + # LLVM_CXXFLAGS - C++ compiler flags for files that include LLVM headers. + # LLVM_FOUND - True if LLVM was found. + + # llvm_map_components_to_libraries - Maps LLVM used components to required libraries. + # Usage: llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES core jit interpreter native ...) + + find_program(LLVM_CONFIG_EXECUTABLE + NAMES llvm-config + PATHS $ENV{LLVM_ROOT}/bin) + + mark_as_advanced(LLVM_CONFIG_EXECUTABLE) + + if(NOT LLVM_CONFIG_EXECUTABLE) + message(FATAL_ERROR "Cannot find LLVM (looking for `llvm-config`). Please, provide LLVM_ROOT environment variable.") + else() + set(LLVM_FOUND TRUE) + + execute_process( + COMMAND ${LLVM_CONFIG_EXECUTABLE} --version + OUTPUT_VARIABLE LLVM_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(LLVM_VERSION VERSION_LESS "5") + message(FATAL_ERROR "LLVM 5+ is required.") + endif() + + execute_process( + COMMAND ${LLVM_CONFIG_EXECUTABLE} --includedir + OUTPUT_VARIABLE LLVM_INCLUDE_DIRS + OUTPUT_STRIP_TRAILING_WHITESPACE) + + execute_process( + COMMAND ${LLVM_CONFIG_EXECUTABLE} --libdir + OUTPUT_VARIABLE LLVM_LIBRARY_DIRS + OUTPUT_STRIP_TRAILING_WHITESPACE) + + execute_process( + COMMAND ${LLVM_CONFIG_EXECUTABLE} --cxxflags + OUTPUT_VARIABLE LLVM_CXXFLAGS + OUTPUT_STRIP_TRAILING_WHITESPACE) + + execute_process( + COMMAND ${LLVM_CONFIG_EXECUTABLE} --targets-built + OUTPUT_VARIABLE LLVM_TARGETS_BUILT + OUTPUT_STRIP_TRAILING_WHITESPACE) + + string(REPLACE " " ";" LLVM_TARGETS_BUILT "${LLVM_TARGETS_BUILT}") + + # Get the link libs we need. + function(llvm_map_components_to_libraries RESULT) + execute_process( + COMMAND ${LLVM_CONFIG_EXECUTABLE} --libs ${ARGN} + OUTPUT_VARIABLE _tmp + OUTPUT_STRIP_TRAILING_WHITESPACE) + + string(REPLACE " " ";" _libs_module "${_tmp}") + + message(STATUS "LLVM Libraries for '${ARGN}': ${_libs_module}") + + execute_process( + COMMAND ${LLVM_CONFIG_EXECUTABLE} --system-libs ${ARGN} + OUTPUT_VARIABLE _libs_system + OUTPUT_STRIP_TRAILING_WHITESPACE) + + string(REPLACE "\n" " " _libs_system "${_libs_system}") + string(REPLACE " " " " _libs_system "${_libs_system}") + string(REPLACE " " ";" _libs_system "${_libs_system}") + + set(${RESULT} ${_libs_module} ${_libs_system} PARENT_SCOPE) + endfunction(llvm_map_components_to_libraries) + + message(STATUS "LLVM Include Directory: ${LLVM_INCLUDE_DIRS}") + message(STATUS "LLVM Library Directory: ${LLVM_LIBRARY_DIRS}") + message(STATUS "LLVM C++ Compiler: ${LLVM_CXXFLAGS}") + endif() +endif() diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index 3c0a3c8899c4..aa7f75182d02 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -8,6 +8,7 @@ #cmakedefine01 USE_VECTORCLASS #cmakedefine01 USE_RDKAFKA #cmakedefine01 USE_CAPNP +#cmakedefine01 USE_EMBEDDED_COMPILER #cmakedefine01 Poco_DataODBC_FOUND #cmakedefine01 Poco_MongoDB_FOUND #cmakedefine01 Poco_NetSSL_FOUND diff --git a/dbms/src/Interpreters/CMakeLists.txt b/dbms/src/Interpreters/CMakeLists.txt index 4a540c128616..f1db16067864 100644 --- a/dbms/src/Interpreters/CMakeLists.txt +++ b/dbms/src/Interpreters/CMakeLists.txt @@ -5,17 +5,14 @@ else () set (PATH_SHARE "/usr/share" CACHE STRING "") endif () -if (INTERNAL_COMPILER_SAME) - set (INTERNAL_COMPILER_EXECUTABLE "${CMAKE_CXX_COMPILER}" CACHE STRING "") -else () - set (INTERNAL_COMPILER_EXECUTABLE "${PATH_SHARE}/clickhouse/bin/clang" CACHE STRING "") +if (USE_EMBEDDED_COMPILER) + set (INTERNAL_COMPILER_EXECUTABLE "clickhouse-clang" CACHE STRING "") + set (INTERNAL_LINKER_EXECUTABLE "clickhouse-lld" CACHE STRING "") endif () set (INTERNAL_COMPILER_NO_WARNING OFF CACHE BOOL "") -#original string: -# -x c++ -std=gnu++1y -O3 -g -Wall -Werror -Wnon-virtual-dtor -march=native -msse4 -mpopcnt -D NDEBUG -D_GLIBCXX_USE_CXX11_ABI=0 -shared -fPIC -fvisibility=hidden -fno-implement-inlines -set (INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -shared -fPIC -fvisibility=hidden -fno-implement-inlines " CACHE STRING "") +set (INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines " CACHE STRING "") string(REPLACE "-no-pie" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) if (INTERNAL_COMPILER_NO_WARNING) string (REPLACE "-Wall" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) diff --git a/dbms/src/Interpreters/Compiler.cpp b/dbms/src/Interpreters/Compiler.cpp index fde9e7176905..3e017d7be851 100644 --- a/dbms/src/Interpreters/Compiler.cpp +++ b/dbms/src/Interpreters/Compiler.cpp @@ -203,6 +203,7 @@ void Compiler::compile( std::string prefix = path + "/" + file_name; std::string cpp_file_path = prefix + ".cpp"; + std::string o_file_path = prefix + ".o"; std::string so_file_path = prefix + ".so"; std::string so_tmp_file_path = prefix + ".so.tmp"; @@ -217,29 +218,35 @@ void Compiler::compile( /// Slightly unconvenient. command << - "LD_LIBRARY_PATH=" PATH_SHARE "/clickhouse/bin/" - " " INTERNAL_COMPILER_EXECUTABLE - " -B " PATH_SHARE "/clickhouse/bin/" - " " INTERNAL_COMPILER_FLAGS -#if INTERNAL_COMPILER_CUSTOM_ROOT - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/c++/*/" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include/" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/clang/*/include/" -#endif - " -I " INTERNAL_COMPILER_HEADERS "/dbms/src/" - " -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/" - " -I " INTERNAL_COMPILER_HEADERS "/contrib/libpcg-random/include/" - " -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR - " -I " INTERNAL_Poco_Foundation_INCLUDE_DIR - " -I " INTERNAL_Boost_INCLUDE_DIRS - " -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/" - " " << additional_compiler_flags << - " -o " << so_tmp_file_path << " " << cpp_file_path - << " 2>&1 || echo Exit code: $?"; + "(" + INTERNAL_COMPILER_EXECUTABLE + " " INTERNAL_COMPILER_FLAGS + #if INTERNAL_COMPILER_CUSTOM_ROOT + " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/" + " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/" + " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/" + " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/" + " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/c++/*/" + " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include/" + " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/clang/*/include/" + #endif + " -I " INTERNAL_COMPILER_HEADERS "/dbms/src/" + " -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/" + " -I " INTERNAL_COMPILER_HEADERS "/contrib/libpcg-random/include/" + " -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR + " -I " INTERNAL_Poco_Foundation_INCLUDE_DIR + " -I " INTERNAL_Boost_INCLUDE_DIRS + " -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/" + " " << additional_compiler_flags << + " -c -o " << o_file_path << " " << cpp_file_path + << " 2>&1" + ") && (" + INTERNAL_LINKER_EXECUTABLE + " -shared" + " -o " << so_tmp_file_path + << " " << o_file_path + << " 2>&1" + ") || echo Return code: $?"; std::string compile_result; @@ -254,6 +261,7 @@ void Compiler::compile( /// If there was an error before, the file with the code remains for viewing. Poco::File(cpp_file_path).remove(); + Poco::File(o_file_path).remove(); Poco::File(so_tmp_file_path).renameTo(so_file_path); SharedLibraryPtr lib(new SharedLibrary(so_file_path)); diff --git a/dbms/src/Interpreters/config_compile.h.in b/dbms/src/Interpreters/config_compile.h.in index 3ef4ef643811..0655cfd95dc5 100644 --- a/dbms/src/Interpreters/config_compile.h.in +++ b/dbms/src/Interpreters/config_compile.h.in @@ -3,6 +3,7 @@ #cmakedefine PATH_SHARE "@PATH_SHARE@" #cmakedefine INTERNAL_COMPILER_FLAGS "@INTERNAL_COMPILER_FLAGS@" #cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@" +#cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@" #cmakedefine INTERNAL_COMPILER_HEADERS "@INTERNAL_COMPILER_HEADERS@" #cmakedefine INTERNAL_COMPILER_HEADERS_ROOT "@INTERNAL_COMPILER_HEADERS_ROOT@" #cmakedefine01 INTERNAL_COMPILER_CUSTOM_ROOT diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 561e6f2e8940..54f99bd3b6e0 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -43,7 +43,6 @@ target_link_libraries (clickhouse-compressor-lib dbms ${Boost_PROGRAM_OPTIONS_LI add_library (clickhouse-format Format.cpp) target_link_libraries (clickhouse-format dbms ${Boost_PROGRAM_OPTIONS_LIBRARY}) - add_executable(clickhouse main.cpp) target_include_directories(clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) target_link_libraries(clickhouse @@ -55,6 +54,12 @@ target_link_libraries(clickhouse clickhouse-extract-from-config clickhouse-compressor-lib clickhouse-format) + +if (USE_EMBEDDED_COMPILER) + add_subdirectory (Compiler) + target_link_libraries (clickhouse clickhouse-compiler) +endif () + INSTALL(TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) # make symbolic links to concrete clickhouse applications macro(install_symlink_to_clickhouse app) @@ -68,6 +73,11 @@ install_symlink_to_clickhouse(clickhouse-benchmark) install_symlink_to_clickhouse(clickhouse-performance-test) install_symlink_to_clickhouse(clickhouse-format) +if (USE_EMBEDDED_COMPILER) + install_symlink_to_clickhouse(clickhouse-clang) + install_symlink_to_clickhouse(clickhouse-lld) +endif () + INSTALL( FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server diff --git a/dbms/src/Server/Compiler/CMakeLists.txt b/dbms/src/Server/Compiler/CMakeLists.txt new file mode 100644 index 000000000000..876786523a35 --- /dev/null +++ b/dbms/src/Server/Compiler/CMakeLists.txt @@ -0,0 +1,40 @@ +add_library(clickhouse-compiler + driver.cpp + cc1_main.cpp + cc1as_main.cpp + lld.cpp) + +target_compile_options(clickhouse-compiler PRIVATE -fno-rtti) + +llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all) + +# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency. +list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz") + +# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. + +target_link_libraries(clickhouse-compiler + +clangBasic clangCodeGen clangDriver clangFrontend clangFrontendTool +clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend +clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers +clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic + +lldCOFF +lldDriver +lldELF +lldMinGW +lldMachO +lldReaderWriter +lldYAML +lldCommon +lldCore + +${REQUIRED_LLVM_LIBRARIES} + +Polly +PollyISL +PollyPPCG + +libtinfo.a +common) diff --git a/dbms/src/Server/Compiler/LICENSE.TXT b/dbms/src/Server/Compiler/LICENSE.TXT new file mode 100644 index 000000000000..b452ca2efd8f --- /dev/null +++ b/dbms/src/Server/Compiler/LICENSE.TXT @@ -0,0 +1,63 @@ +============================================================================== +LLVM Release License +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + +The following pieces of software have additional or alternate copyrights, +licenses, and/or restrictions: + +Program Directory +------- --------- + + diff --git a/dbms/src/Server/Compiler/cc1_main.cpp b/dbms/src/Server/Compiler/cc1_main.cpp new file mode 100644 index 000000000000..1a16746d589d --- /dev/null +++ b/dbms/src/Server/Compiler/cc1_main.cpp @@ -0,0 +1,239 @@ +//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang -cc1 functionality, which implements the +// core compiler functionality along with a number of additional tools for +// demonstration and testing purposes. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Option/Arg.h" +#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" +#include "clang/Config/config.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticBuffer.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "clang/FrontendTool/Utils.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/LinkAllPasses.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include + +#ifdef CLANG_HAVE_RLIMITS +#include +#endif + +using namespace clang; +using namespace llvm::opt; + +//===----------------------------------------------------------------------===// +// Main driver +//===----------------------------------------------------------------------===// + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool GenCrashDiag) { + DiagnosticsEngine &Diags = *static_cast(UserData); + + Diags.Report(diag::err_fe_error_backend) << Message; + + // Run the interrupt handlers to make sure any special cleanups get done, in + // particular that we remove files registered with RemoveFileOnSignal. + llvm::sys::RunInterruptHandlers(); + + // We cannot recover from llvm errors. When reporting a fatal error, exit + // with status 70 to generate crash diagnostics. For BSD systems this is + // defined as an internal software error. Otherwise, exit with status 1. + exit(GenCrashDiag ? 70 : 1); +} + +#ifdef LINK_POLLY_INTO_TOOLS +namespace polly { +void initializePollyPasses(llvm::PassRegistry &Registry); +} +#endif + +#ifdef CLANG_HAVE_RLIMITS +// The amount of stack we think is "sufficient". If less than this much is +// available, we may be unable to reach our template instantiation depth +// limit and other similar limits. +// FIXME: Unify this with the stack we request when spawning a thread to build +// a module. +static const int kSufficientStack = 8 << 20; + +#if defined(__linux__) && defined(__PIE__) +static size_t getCurrentStackAllocation() { + // If we can't compute the current stack usage, allow for 512K of command + // line arguments and environment. + size_t Usage = 512 * 1024; + if (FILE *StatFile = fopen("/proc/self/stat", "r")) { + // We assume that the stack extends from its current address to the end of + // the environment space. In reality, there is another string literal (the + // program name) after the environment, but this is close enough (we only + // need to be within 100K or so). + unsigned long StackPtr, EnvEnd; + // Disable silly GCC -Wformat warning that complains about length + // modifiers on ignored format specifiers. We want to retain these + // for documentation purposes even though they have no effect. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat" +#endif + if (fscanf(StatFile, + "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " + "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " + "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " + "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", + &StackPtr, &EnvEnd) == 2) { +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; + } + fclose(StatFile); + } + return Usage; +} + +#include + +LLVM_ATTRIBUTE_NOINLINE +static void ensureStackAddressSpace(int ExtraChunks = 0) { + // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary + // relatively close to the stack (they are only guaranteed to be 128MiB + // apart). This results in crashes if we happen to heap-allocate more than + // 128MiB before we reach our stack high-water mark. + // + // To avoid these crashes, ensure that we have sufficient virtual memory + // pages allocated before we start running. + size_t Curr = getCurrentStackAllocation(); + const int kTargetStack = kSufficientStack - 256 * 1024; + if (Curr < kTargetStack) { + volatile char *volatile Alloc = + static_cast(alloca(kTargetStack - Curr)); + Alloc[0] = 0; + Alloc[kTargetStack - Curr - 1] = 0; + } +} +#else +static void ensureStackAddressSpace() {} +#endif + +/// Attempt to ensure that we have at least 8MiB of usable stack space. +static void ensureSufficientStack() { + struct rlimit rlim; + if (getrlimit(RLIMIT_STACK, &rlim) != 0) + return; + + // Increase the soft stack limit to our desired level, if necessary and + // possible. + if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { + // Try to allocate sufficient stack. + if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) + rlim.rlim_cur = kSufficientStack; + else if (rlim.rlim_cur == rlim.rlim_max) + return; + else + rlim.rlim_cur = rlim.rlim_max; + + if (setrlimit(RLIMIT_STACK, &rlim) != 0 || + rlim.rlim_cur != kSufficientStack) + return; + } + + // We should now have a stack of size at least kSufficientStack. Ensure + // that we can actually use that much, if necessary. + ensureStackAddressSpace(); +} +#else +static void ensureSufficientStack() {} +#endif + +int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { + ensureSufficientStack(); + + std::unique_ptr Clang(new CompilerInstance()); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + // Register the support for object-file-wrapped Clang modules. + auto PCHOps = Clang->getPCHContainerOperations(); + PCHOps->registerWriter(llvm::make_unique()); + PCHOps->registerReader(llvm::make_unique()); + + // Initialize targets first, so that --version shows registered targets. + llvm::InitializeAllTargets(); + llvm::InitializeAllTargetMCs(); + llvm::InitializeAllAsmPrinters(); + llvm::InitializeAllAsmParsers(); + +#ifdef LINK_POLLY_INTO_TOOLS + llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); + polly::initializePollyPasses(Registry); +#endif + + // Buffer diagnostics from argument parsing so that we can output them using a + // well formed diagnostic object. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + bool Success = CompilerInvocation::CreateFromArgs( + Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); + + // Infer the builtin include path if unspecified. + if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && + Clang->getHeaderSearchOpts().ResourceDir.empty()) + Clang->getHeaderSearchOpts().ResourceDir = + CompilerInvocation::GetResourcesPath(Argv0, MainAddr); + + // Create the actual diagnostics engine. + Clang->createDiagnostics(); + if (!Clang->hasDiagnostics()) + return 1; + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + llvm::install_fatal_error_handler(LLVMErrorHandler, + static_cast(&Clang->getDiagnostics())); + + DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); + if (!Success) + return 1; + + // Execute the frontend actions. + Success = ExecuteCompilerInvocation(Clang.get()); + + // If any timers were active but haven't been destroyed yet, print their + // results now. This happens in -disable-free mode. + llvm::TimerGroup::printAll(llvm::errs()); + + // Our error handler depends on the Diagnostics object, which we're + // potentially about to delete. Uninstall the handler now so that any + // later errors use the default handling behavior instead. + llvm::remove_fatal_error_handler(); + + // When running with -disable-free, don't do any destruction or shutdown. + if (Clang->getFrontendOpts().DisableFree) { + BuryPointer(std::move(Clang)); + return !Success; + } + + return !Success; +} diff --git a/dbms/src/Server/Compiler/cc1as_main.cpp b/dbms/src/Server/Compiler/cc1as_main.cpp new file mode 100644 index 000000000000..e7ab61c96c60 --- /dev/null +++ b/dbms/src/Server/Compiler/cc1as_main.cpp @@ -0,0 +1,541 @@ +//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang -cc1as functionality, which implements +// the direct interface to the LLVM MC based assembler. +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Triple.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCParser/MCAsmParser.h" +#include "llvm/MC/MCParser/MCTargetAsmParser.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MCTargetOptions.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +using namespace clang; +using namespace clang::driver; +using namespace clang::driver::options; +using namespace llvm; +using namespace llvm::opt; + +namespace { + +/// \brief Helper class for representing a single invocation of the assembler. +struct AssemblerInvocation { + /// @name Target Options + /// @{ + + /// The name of the target triple to assemble for. + std::string Triple; + + /// If given, the name of the target CPU to determine which instructions + /// are legal. + std::string CPU; + + /// The list of target specific features to enable or disable -- this should + /// be a list of strings starting with '+' or '-'. + std::vector Features; + + /// The list of symbol definitions. + std::vector SymbolDefs; + + /// @} + /// @name Language Options + /// @{ + + std::vector IncludePaths; + unsigned NoInitialTextSection : 1; + unsigned SaveTemporaryLabels : 1; + unsigned GenDwarfForAssembly : 1; + unsigned RelaxELFRelocations : 1; + unsigned DwarfVersion; + std::string DwarfDebugFlags; + std::string DwarfDebugProducer; + std::string DebugCompilationDir; + llvm::DebugCompressionType CompressDebugSections = + llvm::DebugCompressionType::None; + std::string MainFileName; + + /// @} + /// @name Frontend Options + /// @{ + + std::string InputFile; + std::vector LLVMArgs; + std::string OutputPath; + enum FileType { + FT_Asm, ///< Assembly (.s) output, transliterate mode. + FT_Null, ///< No output, for timing purposes. + FT_Obj ///< Object file output. + }; + FileType OutputType; + unsigned ShowHelp : 1; + unsigned ShowVersion : 1; + + /// @} + /// @name Transliterate Options + /// @{ + + unsigned OutputAsmVariant; + unsigned ShowEncoding : 1; + unsigned ShowInst : 1; + + /// @} + /// @name Assembler Options + /// @{ + + unsigned RelaxAll : 1; + unsigned NoExecStack : 1; + unsigned FatalWarnings : 1; + unsigned IncrementalLinkerCompatible : 1; + + /// The name of the relocation model to use. + std::string RelocationModel; + + /// @} + +public: + AssemblerInvocation() { + Triple = ""; + NoInitialTextSection = 0; + InputFile = "-"; + OutputPath = "-"; + OutputType = FT_Asm; + OutputAsmVariant = 0; + ShowInst = 0; + ShowEncoding = 0; + RelaxAll = 0; + NoExecStack = 0; + FatalWarnings = 0; + IncrementalLinkerCompatible = 0; + DwarfVersion = 0; + } + + static bool CreateFromArgs(AssemblerInvocation &Res, + ArrayRef Argv, + DiagnosticsEngine &Diags); +}; + +} + +bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, + ArrayRef Argv, + DiagnosticsEngine &Diags) { + bool Success = true; + + // Parse the arguments. + std::unique_ptr OptTbl(createDriverOptTable()); + + const unsigned IncludedFlagsBitmask = options::CC1AsOption; + unsigned MissingArgIndex, MissingArgCount; + InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, + IncludedFlagsBitmask); + + // Check for missing argument error. + if (MissingArgCount) { + Diags.Report(diag::err_drv_missing_argument) + << Args.getArgString(MissingArgIndex) << MissingArgCount; + Success = false; + } + + // Issue errors on unknown arguments. + for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { + Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); + Success = false; + } + + // Construct the invocation. + + // Target Options + Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); + Opts.CPU = Args.getLastArgValue(OPT_target_cpu); + Opts.Features = Args.getAllArgValues(OPT_target_feature); + + // Use the default target triple if unspecified. + if (Opts.Triple.empty()) + Opts.Triple = llvm::sys::getDefaultTargetTriple(); + + // Language Options + Opts.IncludePaths = Args.getAllArgValues(OPT_I); + Opts.NoInitialTextSection = Args.hasArg(OPT_n); + Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); + // Any DebugInfoKind implies GenDwarfForAssembly. + Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); + + if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, + OPT_compress_debug_sections_EQ)) { + if (A->getOption().getID() == OPT_compress_debug_sections) { + // TODO: be more clever about the compression type auto-detection + Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; + } else { + Opts.CompressDebugSections = + llvm::StringSwitch(A->getValue()) + .Case("none", llvm::DebugCompressionType::None) + .Case("zlib", llvm::DebugCompressionType::Z) + .Case("zlib-gnu", llvm::DebugCompressionType::GNU) + .Default(llvm::DebugCompressionType::None); + } + } + + Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); + Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); + Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); + Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); + Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); + Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); + + // Frontend Options + if (Args.hasArg(OPT_INPUT)) { + bool First = true; + for (const Arg *A : Args.filtered(OPT_INPUT)) { + if (First) { + Opts.InputFile = A->getValue(); + First = false; + } else { + Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); + Success = false; + } + } + } + Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); + Opts.OutputPath = Args.getLastArgValue(OPT_o); + if (Arg *A = Args.getLastArg(OPT_filetype)) { + StringRef Name = A->getValue(); + unsigned OutputType = StringSwitch(Name) + .Case("asm", FT_Asm) + .Case("null", FT_Null) + .Case("obj", FT_Obj) + .Default(~0U); + if (OutputType == ~0U) { + Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; + Success = false; + } else + Opts.OutputType = FileType(OutputType); + } + Opts.ShowHelp = Args.hasArg(OPT_help); + Opts.ShowVersion = Args.hasArg(OPT_version); + + // Transliterate Options + Opts.OutputAsmVariant = + getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); + Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); + Opts.ShowInst = Args.hasArg(OPT_show_inst); + + // Assemble Options + Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); + Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); + Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); + Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); + Opts.IncrementalLinkerCompatible = + Args.hasArg(OPT_mincremental_linker_compatible); + Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); + + return Success; +} + +static std::unique_ptr +getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, + bool Binary) { + if (Opts.OutputPath.empty()) + Opts.OutputPath = "-"; + + // Make sure that the Out file gets unlinked from the disk if we get a + // SIGINT. + if (Opts.OutputPath != "-") + sys::RemoveFileOnSignal(Opts.OutputPath); + + std::error_code EC; + auto Out = llvm::make_unique( + Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); + if (EC) { + Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath + << EC.message(); + return nullptr; + } + + return Out; +} + +static bool ExecuteAssembler(AssemblerInvocation &Opts, + DiagnosticsEngine &Diags) { + // Get the target specific parser. + std::string Error; + const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); + if (!TheTarget) + return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; + + ErrorOr> Buffer = + MemoryBuffer::getFileOrSTDIN(Opts.InputFile); + + if (std::error_code EC = Buffer.getError()) { + Error = EC.message(); + return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; + } + + SourceMgr SrcMgr; + + // Tell SrcMgr about this buffer, which is what the parser will pick up. + SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); + + // Record the location of the include directories so that the lexer can find + // it later. + SrcMgr.setIncludeDirs(Opts.IncludePaths); + + std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); + assert(MRI && "Unable to create target register info!"); + + std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); + assert(MAI && "Unable to create target asm info!"); + + // Ensure MCAsmInfo initialization occurs before any use, otherwise sections + // may be created with a combination of default and explicit settings. + MAI->setCompressDebugSections(Opts.CompressDebugSections); + + MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); + + bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; + std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); + if (!FDOS) + return true; + + // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and + // MCObjectFileInfo needs a MCContext reference in order to initialize itself. + std::unique_ptr MOFI(new MCObjectFileInfo()); + + MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); + + bool PIC = false; + if (Opts.RelocationModel == "static") { + PIC = false; + } else if (Opts.RelocationModel == "pic") { + PIC = true; + } else { + assert(Opts.RelocationModel == "dynamic-no-pic" && + "Invalid PIC model!"); + PIC = false; + } + + MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); + if (Opts.SaveTemporaryLabels) + Ctx.setAllowTemporaryLabels(false); + if (Opts.GenDwarfForAssembly) + Ctx.setGenDwarfForAssembly(true); + if (!Opts.DwarfDebugFlags.empty()) + Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); + if (!Opts.DwarfDebugProducer.empty()) + Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); + if (!Opts.DebugCompilationDir.empty()) + Ctx.setCompilationDir(Opts.DebugCompilationDir); + if (!Opts.MainFileName.empty()) + Ctx.setMainFileName(StringRef(Opts.MainFileName)); + Ctx.setDwarfVersion(Opts.DwarfVersion); + + // Build up the feature string from the target feature list. + std::string FS; + if (!Opts.Features.empty()) { + FS = Opts.Features[0]; + for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) + FS += "," + Opts.Features[i]; + } + + std::unique_ptr Str; + + std::unique_ptr MCII(TheTarget->createMCInstrInfo()); + std::unique_ptr STI( + TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); + + raw_pwrite_stream *Out = FDOS.get(); + std::unique_ptr BOS; + + // FIXME: There is a bit of code duplication with addPassesToEmitFile. + if (Opts.OutputType == AssemblerInvocation::FT_Asm) { + MCInstPrinter *IP = TheTarget->createMCInstPrinter( + llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); + MCCodeEmitter *CE = nullptr; + MCAsmBackend *MAB = nullptr; + if (Opts.ShowEncoding) { + CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); + MCTargetOptions Options; + MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, Opts.CPU, Options); + } + auto FOut = llvm::make_unique(*Out); + Str.reset(TheTarget->createAsmStreamer( + Ctx, std::move(FOut), /*asmverbose*/ true, + /*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst)); + } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { + Str.reset(createNullStreamer(Ctx)); + } else { + assert(Opts.OutputType == AssemblerInvocation::FT_Obj && + "Invalid file type!"); + if (!FDOS->supportsSeeking()) { + BOS = make_unique(*FDOS); + Out = BOS.get(); + } + + MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); + MCTargetOptions Options; + MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, + Opts.CPU, Options); + Triple T(Opts.Triple); + Str.reset(TheTarget->createMCObjectStreamer( + T, Ctx, std::unique_ptr(MAB), *Out, std::unique_ptr(CE), *STI, + Opts.RelaxAll, Opts.IncrementalLinkerCompatible, + /*DWARFMustBeAtTheEnd*/ true)); + Str.get()->InitSections(Opts.NoExecStack); + } + + bool Failed = false; + + std::unique_ptr Parser( + createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); + + // FIXME: init MCTargetOptions from sanitizer flags here. + MCTargetOptions Options; + std::unique_ptr TAP( + TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); + if (!TAP) + Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; + + // Set values for symbols, if any. + for (auto &S : Opts.SymbolDefs) { + auto Pair = StringRef(S).split('='); + auto Sym = Pair.first; + auto Val = Pair.second; + int64_t Value = 0; + // We have already error checked this in the driver. + Val.getAsInteger(0, Value); + Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); + } + + if (!Failed) { + Parser->setTargetParser(*TAP.get()); + Failed = Parser->Run(Opts.NoInitialTextSection); + } + + // Close Streamer first. + // It might have a reference to the output stream. + Str.reset(); + // Close the output stream early. + BOS.reset(); + FDOS.reset(); + + // Delete output file if there were errors. + if (Failed && Opts.OutputPath != "-") + sys::fs::remove(Opts.OutputPath); + + return Failed; +} + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool GenCrashDiag) { + DiagnosticsEngine &Diags = *static_cast(UserData); + + Diags.Report(diag::err_fe_error_backend) << Message; + + // We cannot recover from llvm errors. + exit(1); +} + +int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { + // Initialize targets and assembly printers/parsers. + InitializeAllTargetInfos(); + InitializeAllTargetMCs(); + InitializeAllAsmParsers(); + + // Construct our diagnostic client. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticPrinter *DiagClient + = new TextDiagnosticPrinter(errs(), &*DiagOpts); + DiagClient->setPrefix("clang -cc1as"); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + ScopedFatalErrorHandler FatalErrorHandler + (LLVMErrorHandler, static_cast(&Diags)); + + // Parse the arguments. + AssemblerInvocation Asm; + if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) + return 1; + + if (Asm.ShowHelp) { + std::unique_ptr Opts(driver::createDriverOptTable()); + Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", + /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, + /*ShowAllAliases=*/false); + return 0; + } + + // Honor -version. + // + // FIXME: Use a better -version message? + if (Asm.ShowVersion) { + llvm::cl::PrintVersionMessage(); + return 0; + } + + // Honor -mllvm. + // + // FIXME: Remove this, one day. + if (!Asm.LLVMArgs.empty()) { + unsigned NumArgs = Asm.LLVMArgs.size(); + auto Args = llvm::make_unique(NumArgs + 2); + Args[0] = "clang (LLVM option parsing)"; + for (unsigned i = 0; i != NumArgs; ++i) + Args[i + 1] = Asm.LLVMArgs[i].c_str(); + Args[NumArgs + 1] = nullptr; + llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); + } + + // Execute the invocation, unless there were parsing errors. + bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); + + // If any timers were active but haven't been destroyed yet, print their + // results now. + TimerGroup::printAll(errs()); + + return !!Failed; +} diff --git a/dbms/src/Server/Compiler/driver.cpp b/dbms/src/Server/Compiler/driver.cpp new file mode 100644 index 000000000000..30511b8253a8 --- /dev/null +++ b/dbms/src/Server/Compiler/driver.cpp @@ -0,0 +1,520 @@ +//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang driver; it is a thin wrapper +// for functionality in the Driver clang library. +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Driver/ToolChain.h" +#include "clang/Frontend/ChainedDiagnosticConsumer.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/SerializedDiagnosticPrinter.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +using namespace clang; +using namespace clang::driver; +using namespace llvm::opt; + +std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { + if (!CanonicalPrefixes) { + SmallString<128> ExecutablePath(Argv0); + // Do a PATH lookup if Argv0 isn't a valid path. + if (!llvm::sys::fs::exists(ExecutablePath)) + if (llvm::ErrorOr P = + llvm::sys::findProgramByName(ExecutablePath)) + ExecutablePath = *P; + return ExecutablePath.str(); + } + + // This just needs to be some symbol in the binary; C++ doesn't + // allow taking the address of ::main however. + void *P = (void*) (intptr_t) GetExecutablePath; + return llvm::sys::fs::getMainExecutable(Argv0, P); +} + +static const char *GetStableCStr(std::set &SavedStrings, + StringRef S) { + return SavedStrings.insert(S).first->c_str(); +} + +/// ApplyQAOverride - Apply a list of edits to the input argument lists. +/// +/// The input string is a space separate list of edits to perform, +/// they are applied in order to the input argument lists. Edits +/// should be one of the following forms: +/// +/// '#': Silence information about the changes to the command line arguments. +/// +/// '^': Add FOO as a new argument at the beginning of the command line. +/// +/// '+': Add FOO as a new argument at the end of the command line. +/// +/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command +/// line. +/// +/// 'xOPTION': Removes all instances of the literal argument OPTION. +/// +/// 'XOPTION': Removes all instances of the literal argument OPTION, +/// and the following argument. +/// +/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' +/// at the end of the command line. +/// +/// \param OS - The stream to write edit information to. +/// \param Args - The vector of command line arguments. +/// \param Edit - The override command to perform. +/// \param SavedStrings - Set to use for storing string representations. +static void ApplyOneQAOverride(raw_ostream &OS, + SmallVectorImpl &Args, + StringRef Edit, + std::set &SavedStrings) { + // This does not need to be efficient. + + if (Edit[0] == '^') { + const char *Str = + GetStableCStr(SavedStrings, Edit.substr(1)); + OS << "### Adding argument " << Str << " at beginning\n"; + Args.insert(Args.begin() + 1, Str); + } else if (Edit[0] == '+') { + const char *Str = + GetStableCStr(SavedStrings, Edit.substr(1)); + OS << "### Adding argument " << Str << " at end\n"; + Args.push_back(Str); + } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && + Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { + StringRef MatchPattern = Edit.substr(2).split('/').first; + StringRef ReplPattern = Edit.substr(2).split('/').second; + ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); + + for (unsigned i = 1, e = Args.size(); i != e; ++i) { + // Ignore end-of-line response file markers + if (Args[i] == nullptr) + continue; + std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); + + if (Repl != Args[i]) { + OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; + Args[i] = GetStableCStr(SavedStrings, Repl); + } + } + } else if (Edit[0] == 'x' || Edit[0] == 'X') { + auto Option = Edit.substr(1); + for (unsigned i = 1; i < Args.size();) { + if (Option == Args[i]) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + if (Edit[0] == 'X') { + if (i < Args.size()) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + } else + OS << "### Invalid X edit, end of command line!\n"; + } + } else + ++i; + } + } else if (Edit[0] == 'O') { + for (unsigned i = 1; i < Args.size();) { + const char *A = Args[i]; + // Ignore end-of-line response file markers + if (A == nullptr) + continue; + if (A[0] == '-' && A[1] == 'O' && + (A[2] == '\0' || + (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || + ('0' <= A[2] && A[2] <= '9'))))) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + } else + ++i; + } + OS << "### Adding argument " << Edit << " at end\n"; + Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); + } else { + OS << "### Unrecognized edit: " << Edit << "\n"; + } +} + +/// ApplyQAOverride - Apply a comma separate list of edits to the +/// input argument lists. See ApplyOneQAOverride. +static void ApplyQAOverride(SmallVectorImpl &Args, + const char *OverrideStr, + std::set &SavedStrings) { + raw_ostream *OS = &llvm::errs(); + + if (OverrideStr[0] == '#') { + ++OverrideStr; + OS = &llvm::nulls(); + } + + *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; + + // This does not need to be efficient. + + const char *S = OverrideStr; + while (*S) { + const char *End = ::strchr(S, ' '); + if (!End) + End = S + strlen(S); + if (End != S) + ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); + S = End; + if (*S != '\0') + ++S; + } +} + +extern int cc1_main(ArrayRef Argv, const char *Argv0, + void *MainAddr); +extern int cc1as_main(ArrayRef Argv, const char *Argv0, + void *MainAddr); + +static void insertTargetAndModeArgs(const ParsedClangName &NameParts, + SmallVectorImpl &ArgVector, + std::set &SavedStrings) { + // Put target and mode arguments at the start of argument list so that + // arguments specified in command line could override them. Avoid putting + // them at index 0, as an option like '-cc1' must remain the first. + auto InsertionPoint = ArgVector.begin(); + if (InsertionPoint != ArgVector.end()) + ++InsertionPoint; + + if (NameParts.DriverMode) { + // Add the mode flag to the arguments. + ArgVector.insert(InsertionPoint, + GetStableCStr(SavedStrings, NameParts.DriverMode)); + } + + if (NameParts.TargetIsValid) { + const char *arr[] = {"-target", GetStableCStr(SavedStrings, + NameParts.TargetPrefix)}; + ArgVector.insert(InsertionPoint, std::begin(arr), std::end(arr)); + } +} + +static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, + SmallVectorImpl &Opts) { + llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); + // The first instance of '#' should be replaced with '=' in each option. + for (const char *Opt : Opts) + if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) + *NumberSignPtr = '='; +} + +static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { + // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. + TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); + if (TheDriver.CCPrintOptions) + TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); + + // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. + TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); + if (TheDriver.CCPrintHeaders) + TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); + + // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. + TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); + if (TheDriver.CCLogDiagnostics) + TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); +} + +static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, + const std::string &Path) { + // If the clang binary happens to be named cl.exe for compatibility reasons, + // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. + StringRef ExeBasename(llvm::sys::path::filename(Path)); + if (ExeBasename.equals_lower("cl.exe")) + ExeBasename = "clang-cl.exe"; + DiagClient->setPrefix(ExeBasename); +} + +// This lets us create the DiagnosticsEngine with a properly-filled-out +// DiagnosticOptions instance. +static DiagnosticOptions * +CreateAndPopulateDiagOpts(ArrayRef argv) { + auto *DiagOpts = new DiagnosticOptions; + std::unique_ptr Opts(createDriverOptTable()); + unsigned MissingArgIndex, MissingArgCount; + InputArgList Args = + Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); + // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. + // Any errors that would be diagnosed here will also be diagnosed later, + // when the DiagnosticsEngine actually exists. + (void)ParseDiagnosticArgs(*DiagOpts, Args); + return DiagOpts; +} + +static void SetInstallDir(SmallVectorImpl &argv, + Driver &TheDriver, bool CanonicalPrefixes) { + // Attempt to find the original path used to invoke the driver, to determine + // the installed path. We do this manually, because we want to support that + // path being a symlink. + SmallString<128> InstalledPath(argv[0]); + + // Do a PATH lookup, if there are no directory components. + if (llvm::sys::path::filename(InstalledPath) == InstalledPath) + if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( + llvm::sys::path::filename(InstalledPath.str()))) + InstalledPath = *Tmp; + + // FIXME: We don't actually canonicalize this, we just make it absolute. + if (CanonicalPrefixes) + llvm::sys::fs::make_absolute(InstalledPath); + + StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); + if (llvm::sys::fs::exists(InstalledPathParent)) + TheDriver.setInstalledDir(InstalledPathParent); +} + +static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { + void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; + if (Tool == "") + return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); + if (Tool == "as") + return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); + + // Reject unknown tools. + llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n"; + return 1; +} + +int mainEntryClickHouseClang(int argc_, char **argv_) { + llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]); + llvm::PrettyStackTraceProgram X(argc_, argv_); + llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. + + if (llvm::sys::Process::FixupStandardFileDescriptors()) + return 1; + + SmallVector argv; + llvm::SpecificBumpPtrAllocator ArgAllocator; + std::error_code EC = llvm::sys::Process::GetArgumentVector( + argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator); + if (EC) { + llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n'; + return 1; + } + + llvm::InitializeAllTargets(); + auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); + + llvm::BumpPtrAllocator A; + llvm::StringSaver Saver(A); + + // Parse response files using the GNU syntax, unless we're in CL mode. There + // are two ways to put clang in CL compatibility mode: argv[0] is either + // clang-cl or cl, or --driver-mode=cl is on the command line. The normal + // command line parsing can't happen until after response file parsing, so we + // have to manually search for a --driver-mode=cl argument the hard way. + // Finally, our -cc1 tools don't care which tokenization mode we use because + // response files written by clang will tokenize the same way in either mode. + bool ClangCLMode = false; + if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || + std::find_if(argv.begin(), argv.end(), [](const char *F) { + return F && strcmp(F, "--driver-mode=cl") == 0; + }) != argv.end()) { + ClangCLMode = true; + } + enum { Default, POSIX, Windows } RSPQuoting = Default; + for (const char *F : argv) { + if (strcmp(F, "--rsp-quoting=posix") == 0) + RSPQuoting = POSIX; + else if (strcmp(F, "--rsp-quoting=windows") == 0) + RSPQuoting = Windows; + } + + // Determines whether we want nullptr markers in argv to indicate response + // files end-of-lines. We only use this for the /LINK driver argument with + // clang-cl.exe on Windows. + bool MarkEOLs = ClangCLMode; + + llvm::cl::TokenizerCallback Tokenizer; + if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) + Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; + else + Tokenizer = &llvm::cl::TokenizeGNUCommandLine; + + if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) + MarkEOLs = false; + llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); + + // Handle -cc1 integrated tools, even if -cc1 was expanded from a response + // file. + auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), + [](const char *A) { return A != nullptr; }); + if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { + // If -cc1 came from a response file, remove the EOL sentinels. + if (MarkEOLs) { + auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); + argv.resize(newEnd - argv.begin()); + } + return ExecuteCC1Tool(argv, argv[1] + 4); + } + + bool CanonicalPrefixes = true; + for (int i = 1, size = argv.size(); i < size; ++i) { + // Skip end-of-line response file markers + if (argv[i] == nullptr) + continue; + if (StringRef(argv[i]) == "-no-canonical-prefixes") { + CanonicalPrefixes = false; + break; + } + } + + // Handle CL and _CL_ which permits additional command line options to be + // prepended or appended. + if (ClangCLMode) { + // Arguments in "CL" are prepended. + llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); + if (OptCL.hasValue()) { + SmallVector PrependedOpts; + getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); + + // Insert right after the program name to prepend to the argument list. + argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); + } + // Arguments in "_CL_" are appended. + llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); + if (Opt_CL_.hasValue()) { + SmallVector AppendedOpts; + getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); + + // Insert at the end of the argument list to append. + argv.append(AppendedOpts.begin(), AppendedOpts.end()); + } + } + + std::set SavedStrings; + // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the + // scenes. + if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { + // FIXME: Driver shouldn't take extra initial argument. + ApplyQAOverride(argv, OverrideStr, SavedStrings); + } + + std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); + + IntrusiveRefCntPtr DiagOpts = + CreateAndPopulateDiagOpts(argv); + + TextDiagnosticPrinter *DiagClient + = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); + FixupDiagPrefixExeName(DiagClient, Path); + + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); + + if (!DiagOpts->DiagnosticSerializationFile.empty()) { + auto SerializedConsumer = + clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, + &*DiagOpts, /*MergeChildRecords=*/true); + Diags.setClient(new ChainedDiagnosticConsumer( + Diags.takeClient(), std::move(SerializedConsumer))); + } + + ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); + + Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); + SetInstallDir(argv, TheDriver, CanonicalPrefixes); + TheDriver.setTargetAndMode(TargetAndMode); + + insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); + + SetBackdoorDriverOutputsFromEnvVars(TheDriver); + + std::unique_ptr C(TheDriver.BuildCompilation(argv)); + int Res = 1; + if (C && !C->containsError()) { + SmallVector, 4> FailingCommands; + Res = TheDriver.ExecuteCompilation(*C, FailingCommands); + + // Force a crash to test the diagnostics. + if (TheDriver.GenReproducer) { + Diags.Report(diag::err_drv_force_crash) + << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); + + // Pretend that every command failed. + FailingCommands.clear(); + for (const auto &J : C->getJobs()) + if (const Command *C = dyn_cast(&J)) + FailingCommands.push_back(std::make_pair(-1, C)); + } + + for (const auto &P : FailingCommands) { + int CommandRes = P.first; + const Command *FailingCommand = P.second; + if (!Res) + Res = CommandRes; + + // If result status is < 0, then the driver command signalled an error. + // If result status is 70, then the driver command reported a fatal error. + // On Windows, abort will return an exit code of 3. In these cases, + // generate additional diagnostic information if possible. + bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; +#ifdef LLVM_ON_WIN32 + DiagnoseCrash |= CommandRes == 3; +#endif + if (DiagnoseCrash) { + TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); + break; + } + } + } + + Diags.getClient()->finish(); + + // If any timers were active but haven't been destroyed yet, print their + // results now. This happens in -disable-free mode. + llvm::TimerGroup::printAll(llvm::errs()); + +#ifdef LLVM_ON_WIN32 + // Exit status should not be negative on Win32, unless abnormal termination. + // Once abnormal termiation was caught, negative status should not be + // propagated. + if (Res < 0) + Res = 1; +#endif + + // If we have multiple failing commands, we return the result of the first + // failing command. + return Res; +} diff --git a/dbms/src/Server/Compiler/lld.cpp b/dbms/src/Server/Compiler/lld.cpp new file mode 100644 index 000000000000..5c2efcfdcf07 --- /dev/null +++ b/dbms/src/Server/Compiler/lld.cpp @@ -0,0 +1,40 @@ +//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===// +// +// The LLVM Linker +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the lld driver. This is a thin wrapper which +// dispatches to the given platform specific driver. +// +// If there is -flavor option, it is dispatched according to the arguments. +// If the flavor parameter is not present, then it is dispatched according +// to argv[0]. +// +//===----------------------------------------------------------------------===// + +#include "lld/Common/Driver.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Signals.h" + +using namespace lld; +using namespace llvm; +using namespace llvm::sys; + +int mainEntryClickHouseLLD(int Argc, char **Argv) { + // Standard set up, so program fails gracefully. + sys::PrintStackTraceOnErrorSignal(Argv[0]); + PrettyStackTraceProgram StackPrinter(Argc, Argv); + llvm_shutdown_obj Shutdown; + + std::vector Args(Argv, Argv + Argc); + return !elf::link(Args, true); +} diff --git a/dbms/src/Server/main.cpp b/dbms/src/Server/main.cpp index c6fe400e3643..b36465751ae3 100644 --- a/dbms/src/Server/main.cpp +++ b/dbms/src/Server/main.cpp @@ -1,7 +1,10 @@ #include +#include + #if USE_TCMALLOC #include #endif + #include "Server.h" #include "LocalServer.h" #include @@ -16,6 +19,11 @@ int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); int mainEntryClickHouseCompressor(int argc, char ** argv); int mainEntryClickHouseFormat(int argc, char ** argv); +#if USE_EMBEDDED_COMPILER + int mainEntryClickHouseClang(int argc, char ** argv); + int mainEntryClickHouseLLD(int argc, char ** argv); +#endif + namespace { @@ -32,7 +40,11 @@ std::pair clickhouse_applications[] = {"performance-test", mainEntryClickHousePerformanceTest}, {"extract-from-config", mainEntryClickHouseExtractFromConfig}, {"compressor", mainEntryClickHouseCompressor}, - {"format", mainEntryClickHouseFormat} + {"format", mainEntryClickHouseFormat}, +#if USE_EMBEDDED_COMPILER + {"clang", mainEntryClickHouseClang}, + {"lld", mainEntryClickHouseLLD}, +#endif }; @@ -70,6 +82,11 @@ bool isClickhouseApp(const std::string & app_suffix, std::vector & argv) int main(int argc_, char ** argv_) { +#if USE_EMBEDDED_COMPILER + if (argc_ >= 2 && 0 == strcmp(argv_[1], "-cc1")) + return mainEntryClickHouseClang(argc_, argv_); +#endif + #if USE_TCMALLOC /** Without this option, tcmalloc returns memory to OS too frequently for medium-sized memory allocations * (like IO buffers, column vectors, hash tables, etc.), diff --git a/debian/copy_clang_binaries.sh b/debian/copy_clang_binaries.sh deleted file mode 100755 index 1bed815c1a9b..000000000000 --- a/debian/copy_clang_binaries.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -e - -# Копирует бинарник clang а также ld и shared-библиотеку libstdc++ в указанную директорию. -# Так повезло, что этого достаточно, чтобы затем собирать код на удалённом сервере с совпадающей версией Ubuntu, но без установленного компилятора. - -DST=${1:-.}; -PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:$PATH" -LD=$(command -v gold || command -v ld.gold || command -v ld) - -# Should be runned with correct path to clang -if [ -z "$CLANG" ]; then - CLANG=$(which clang) -fi - -if [ ! -x "$CLANG" ]; then - echo "Not found executable clang." - exit 1 -fi - -if [ ! -x "$LD" ]; then - echo "Not found executable gold or ld." - exit 1 -fi - -cp "$CLANG" "${DST}/clang" -cp "$LD" "${DST}/ld" - -STDCPP=$(ldd $CLANG | grep -oE '/[^ ]+libstdc++[^ ]+') - -[ -f "$STDCPP" ] && cp "$STDCPP" $DST diff --git a/debian/rules b/debian/rules index a58556bb5f93..b8a75557f107 100755 --- a/debian/rules +++ b/debian/rules @@ -22,7 +22,6 @@ DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) CMAKE_FLAGS += -DENABLE_TESTS=0 DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8) -# CMAKE_FLAGS += -DINTERNAL_COMPILER_EXECUTABLE=$(basename $(DEB_CLANG)) # TODO: this is actual only if you will also change clang name in copy_clang_binaries.sh #DEB_CC ?= gcc-6 #DEB_CXX ?= g++-6 @@ -85,11 +84,9 @@ override_dh_install: cp debian/clickhouse-server.cron.d $(DESTDIR)/etc/cron.d/clickhouse-server # In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation. - mkdir -p $(DESTDIR)/usr/share/clickhouse/bin $(DESTDIR)/usr/share/clickhouse/headers -ifeq ($(USE_INTERNAL_COMPILER),1) - CLANG=$(DEB_CLANG) debian/copy_clang_binaries.sh $(DESTDIR)/usr/share/clickhouse/bin/ + mkdir -p $(DESTDIR)/usr/share/clickhouse/headers + CLANG=$(DEB_CLANG) ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers -endif # fake metrika files when private dir is empty mkdir -p debian/tmp/etc/clickhouse-server/metrika diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index a567387c065a..f9139d2ccde4 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -1,4 +1,15 @@ enable_language (ASM) -add_library (glibc-compatibility glibc-compatibility.c musl/pipe2.c musl/fallocate.c musl/longjmp.s musl/vasprintf.c musl/lgamma.c) + +add_library (glibc-compatibility glibc-compatibility.c +musl/pipe2.c +musl/fallocate.c +musl/longjmp.s +musl/vasprintf.c +musl/lgamma.c +musl/posix_spawn.c +musl/futimens.c +musl/syscall.s +musl/syscall_ret.c +musl/sched_cpucount.c) add_subdirectory (tests) diff --git a/libs/libglibc-compatibility/glibc-compatibility.c b/libs/libglibc-compatibility/glibc-compatibility.c index 5f9c13783785..33fdc772be44 100644 --- a/libs/libglibc-compatibility/glibc-compatibility.c +++ b/libs/libglibc-compatibility/glibc-compatibility.c @@ -119,6 +119,11 @@ int __open_2(const char *path, int oflag) } +/// No-ops. +int pthread_setname_np(pthread_t thread, const char *name) { return 0; } +int pthread_getname_np(pthread_t thread, char *name, size_t len) { name[0] = '\0'; return 0; }; + + #if defined (__cplusplus) } #endif diff --git a/libs/libglibc-compatibility/musl/futimens.c b/libs/libglibc-compatibility/musl/futimens.c new file mode 100644 index 000000000000..1af26ab74fb7 --- /dev/null +++ b/libs/libglibc-compatibility/musl/futimens.c @@ -0,0 +1,38 @@ +#include +#include +#include +#include +#include "syscall.h" +#include + +int futimens(int fd, const struct timespec times[2]) +{ + int r = __syscall(SYS_utimensat, fd, 0, times, 0); +#ifdef SYS_futimesat + if (r != -ENOSYS) return __syscall_ret(r); + struct timeval *tv = 0, tmp[2]; + if (times) { + int i; + tv = tmp; + for (i=0; i<2; i++) { + if (times[i].tv_nsec >= 1000000000ULL) { + if (times[i].tv_nsec == UTIME_NOW && + times[1-i].tv_nsec == UTIME_NOW) { + tv = 0; + break; + } + if (times[i].tv_nsec == UTIME_OMIT) + return __syscall_ret(-ENOSYS); + return __syscall_ret(-EINVAL); + } + tmp[i].tv_sec = times[i].tv_sec; + tmp[i].tv_usec = times[i].tv_nsec / 1000; + } + } + + r = __syscall(SYS_futimesat, fd, 0, tv); + if (r != -ENOSYS || fd != AT_FDCWD) return __syscall_ret(r); + r = __syscall(SYS_utimes, 0, tv); +#endif + return __syscall_ret(r); +} diff --git a/libs/libglibc-compatibility/musl/pipe2.c b/libs/libglibc-compatibility/musl/pipe2.c index 061f2e073132..09358d8a2bb7 100644 --- a/libs/libglibc-compatibility/musl/pipe2.c +++ b/libs/libglibc-compatibility/musl/pipe2.c @@ -1,24 +1,23 @@ #include #include #include -#include - -extern long int syscall (long int __sysno, ...) __THROW; +#include +#include "syscall.h" int pipe2(int fd[2], int flag) { if (!flag) return pipe(fd); - int ret = syscall(SYS_pipe2, fd, flag); - if (ret != -ENOSYS) return -ret; + int ret = __syscall(SYS_pipe2, fd, flag); + if (ret != -ENOSYS) return __syscall_ret(ret); ret = pipe(fd); if (ret) return ret; if (flag & O_CLOEXEC) { - syscall(SYS_fcntl, fd[0], F_SETFD, FD_CLOEXEC); - syscall(SYS_fcntl, fd[1], F_SETFD, FD_CLOEXEC); + __syscall(SYS_fcntl, fd[0], F_SETFD, FD_CLOEXEC); + __syscall(SYS_fcntl, fd[1], F_SETFD, FD_CLOEXEC); } if (flag & O_NONBLOCK) { - syscall(SYS_fcntl, fd[0], F_SETFL, O_NONBLOCK); - syscall(SYS_fcntl, fd[1], F_SETFL, O_NONBLOCK); + __syscall(SYS_fcntl, fd[0], F_SETFL, O_NONBLOCK); + __syscall(SYS_fcntl, fd[1], F_SETFL, O_NONBLOCK); } return 0; } diff --git a/libs/libglibc-compatibility/musl/posix_spawn.c b/libs/libglibc-compatibility/musl/posix_spawn.c new file mode 100644 index 000000000000..a66e64e01a20 --- /dev/null +++ b/libs/libglibc-compatibility/musl/posix_spawn.c @@ -0,0 +1,107 @@ +/// Very limited implementation. Half of code from Musl was cut. +/// This is Ok, because for now, this function is used only from clang driver. + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "syscall.h" + +struct args { + int p[2]; + sigset_t oldmask; + const char *path; + int (*exec)(const char *, char *const *, char *const *); + const posix_spawn_file_actions_t *fa; + const posix_spawnattr_t *restrict attr; + char *const *argv, *const *envp; +}; + +void __get_handler_set(sigset_t *); + +static int child(void *args_vp) +{ + int ret; + struct args *args = args_vp; + int p = args->p[1]; + const posix_spawnattr_t *restrict attr = args->attr; + + close(args->p[0]); + + /* Close-on-exec flag may have been lost if we moved the pipe + * to a different fd. We don't use F_DUPFD_CLOEXEC above because + * it would fail on older kernels and atomicity is not needed -- + * in this process there are no threads or signal handlers. */ + __syscall(SYS_fcntl, p, F_SETFD, FD_CLOEXEC); + + pthread_sigmask(SIG_SETMASK, (attr->__flags & POSIX_SPAWN_SETSIGMASK) + ? &attr->__ss : &args->oldmask, 0); + + args->exec(args->path, args->argv, args->envp); + ret = -errno; + + /* Since sizeof errno < PIPE_BUF, the write is atomic. */ + ret = -ret; + if (ret) while (__syscall(SYS_write, p, &ret, sizeof ret) < 0); + _exit(127); +} + + +int __posix_spawnx(pid_t *restrict res, const char *restrict path, + int (*exec)(const char *, char *const *, char *const *), + const posix_spawn_file_actions_t *fa, + const posix_spawnattr_t *restrict attr, + char *const argv[restrict], char *const envp[restrict]) +{ + pid_t pid; + char stack[1024]; + int ec=0, cs; + struct args args; + + if (pipe2(args.p, O_CLOEXEC)) + return errno; + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + + args.path = path; + args.exec = exec; + args.fa = fa; + args.attr = attr ? attr : &(const posix_spawnattr_t){0}; + args.argv = argv; + args.envp = envp; + + pid = clone(child, stack+sizeof stack, + CLONE_VM|CLONE_VFORK|SIGCHLD, &args); + close(args.p[1]); + + if (pid > 0) { + if (read(args.p[0], &ec, sizeof ec) != sizeof ec) ec = 0; + else waitpid(pid, &(int){0}, 0); + } else { + ec = -pid; + } + + close(args.p[0]); + + if (!ec && res) *res = pid; + + pthread_setcancelstate(cs, 0); + + return ec; +} + +int posix_spawn(pid_t *restrict res, const char *restrict path, + const posix_spawn_file_actions_t *fa, + const posix_spawnattr_t *restrict attr, + char *const argv[restrict], char *const envp[restrict]) +{ + return __posix_spawnx(res, path, execve, fa, attr, argv, envp); +} diff --git a/libs/libglibc-compatibility/musl/sched_cpucount.c b/libs/libglibc-compatibility/musl/sched_cpucount.c new file mode 100644 index 000000000000..94aa259e6ad5 --- /dev/null +++ b/libs/libglibc-compatibility/musl/sched_cpucount.c @@ -0,0 +1,11 @@ +#define _GNU_SOURCE +#include + +int __sched_cpucount(size_t size, const cpu_set_t *set) +{ + size_t i, j, cnt=0; + const unsigned char *p = (const void *)set; + for (i=0; i +#include "syscall.h" + +long __syscall_ret(unsigned long r) +{ + if (r > -4096UL) { + errno = -r; + return -1; + } + return r; +} diff --git a/release b/release index 255f973eb91f..5a5e82d4ccaf 100755 --- a/release +++ b/release @@ -61,7 +61,7 @@ if [ -z "$THREAD_COUNT" ] ; then THREAD_COUNT=`nproc || grep -c ^processor /proc/cpuinfo` fi -CMAKE_FLAGS+=" $LIBTCMALLOC_OPTS -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE" +CMAKE_FLAGS+=" $LIBTCMALLOC_OPTS -D CMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -D USE_EMBEDDED_COMPILER=1" REVISION+=$VERSION_POSTFIX echo -e "\nCurrent revision is $REVISION" @@ -69,4 +69,6 @@ echo -e "\nCurrent revision is $REVISION" gen_changelog "$REVISION" "" "$AUTHOR" "" # Build (only binary packages). -debuild -e PATH -e SSH_AUTH_SOCK -e DEB_BUILD_OPTIONS=parallel=$THREAD_COUNT -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e USE_INTERNAL_COMPILER=1 -e CMAKE_FLAGS="$CMAKE_FLAGS" -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS} +debuild -e PATH -e SSH_AUTH_SOCK -e DEB_BUILD_OPTIONS=parallel=$THREAD_COUNT \ + -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e CMAKE_FLAGS="$CMAKE_FLAGS" \ + -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS}