Enable Link Time Optimization (LTO) with clang

* Change the LLVM formula to build the LLVMgold plugin
* Pass "-flto" for osquery and all dependencies when building with clang on Unix
* Pass "-flto" to modules when they are built
* Set `ranlib` and `ar` to their LLVM verions when building on Linux

CMake/CMakeLibs.cmake

@@ -283,6 +283,10 @@ macro(ADD_OSQUERY_MODULE TARGET)
     # However it is only provided as an example for unit testing.
     target_link_libraries(${TARGET} "-static-libstdc++")
   endif()
+  if(NOT WINDOWS AND CMAKE_CXX_COMPILER MATCHES "clang")
+    #enable LTO builds of modules when building with clang on Unix
+    target_link_libraries(${TARGET} "-flto")
+  endif()
   set_target_properties(${TARGET} PROPERTIES COMPILE_FLAGS "${CXX_COMPILE_FLAGS}")
   set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${TARGET})
 endmacro(ADD_OSQUERY_MODULE)

CMakeLists.txt

@@ -72,6 +72,12 @@ if(NOT WIN32)
     -fdata-sections
     -ffunction-sections
   )
+  if (CMAKE_CXX_COMPILER MATCHES "clang")
+    # Only enale LTO builds when using clang on Unix, for now
+    add_compile_options(
+      -flto
+    )
+  endif()
 endif()
 
 # osquery additional compiler flags added by CMake.
@@ -142,6 +148,20 @@ else()
   endif()
 endif()
 
+if(UNIX AND NOT APPLE)
+  if (CMAKE_CXX_COMPILER MATCHES "clang")
+    # Use the LLVM versions of ar and ranlib to support LTO builds
+    find_program(LLVMRANLIB_EXECUTABLE "llvm-ranlib" ENV PATH)
+    find_program(LLVMAR_EXECUTABLE "llvm-ar" ENV PATH)
+    set(CMAKE_AR "${LLVMAR_EXECUTABLE}")
+    set(CMAKE_RANLIB "${LLVMRANLIB_EXECUTABLE}")
+    set(CMAKE_C_ARCHIVE_CREATE "${LLVMAR_EXECUTABLE} qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+    set(CMAKE_CXX_ARCHIVE_CREATE "${LLVMAR_EXECUTABLE} qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+    set(CMAKE_C_ARCHIVE_FINISH "${LLVMRANLIB_EXECUTABLE} <TARGET>")
+    set(CMAKE_CXX_ARCHIVE_FINISH "${CMAKE_C_ARCHIVE_FINISH}")
+  endif()
+endif()
+
 # make debug (environment variable from Makefile)
 if(DEFINED ENV{DEBUG})
   set(DEBUG TRUE)
@@ -267,6 +287,12 @@ if(WINDOWS)
   endforeach(flag_var)
 
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /LTCG")
+else()
+  if (CMAKE_CXX_COMPILER MATCHES "clang")
+    # Clang on Unix uses LTO; we also need to pass -flto when linking
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto")
+  endif()
+
 endif()
 
 if(NOT IS_DIRECTORY ${CMAKE_SOURCE_DIR}/third-party/sqlite3)

osquery/CMakeLists.txt

@@ -33,6 +33,12 @@ else()
     -Wno-unused-parameter
     -Wno-gnu-case-range
   )
+  if (CMAKE_CXX_COMPILER MATCHES "clang")
+    # Enable LTO when building with clang on Unix
+    add_compile_options(
+      -flto
+    )
+  endif()
 endif()
 
 if(WINDOWS)
@@ -106,6 +112,12 @@ endif()
 
 # The platform-specific SDK + core linker flags.
 if(NOT WINDOWS)
+  if (CMAKE_CXX_COMPILER MATCHES "clang")
+    # When doing a clang on Unix LTO build, the link step also needs
+    # -flto on the comand line
+    ADD_OSQUERY_LINK_CORE("-flto")
+    ADD_OSQUERY_LINK_ADDITIONAL("-flto")
+  endif()
   ADD_OSQUERY_LINK_CORE("-rdynamic")
 endif()
 if(APPLE)

tools/provision/formula/llvm.rb

@@ -49,11 +49,12 @@ class Llvm < AbstractOsqueryFormula
     end
   end
 
-  bottle do
-    root_url "https://osquery-packages.s3.amazonaws.com/bottles"
-    cellar :any_skip_relocation
-    sha256 "349e1f8f4831665c87245cdc727c34b23f7bbd6b6fccbd7c39a3c3a89b002767" => :x86_64_linux
-  end
+  # The current bottle does not build LLVMgold.so
+  #bottle do
+  #  root_url "https://osquery-packages.s3.amazonaws.com/bottles"
+  #  cellar :any_skip_relocation
+  #  sha256 "349e1f8f4831665c87245cdc727c34b23f7bbd6b6fccbd7c39a3c3a89b002767" => :x86_64_linux
+  #end
 
   head do
     url "http://llvm.org/git/llvm.git"
@@ -87,6 +88,8 @@ class Llvm < AbstractOsqueryFormula
 
   deprecated_option "rtti" => "with-rtti"
 
+  depends_on "binutils" if build.with? "clang"
+
   # Apple's libstdc++ is too old to build LLVM
   fails_with :gcc
   fails_with :llvm
@@ -123,6 +126,12 @@ def install
       args << "-DLLVM_ENABLE_EH=ON"
     end
 
+    if build.with? "clang"
+      # build the LLVMGold plugin
+      binutils = Formula["binutils"].prefix/"include"
+      args << "-DLLVM_BINUTILS_INCDIR=#{binutils}"
+    end
+
     args << "-DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON" if build.with? "compiler-rt"
     args << "-DLLVM_INSTALL_UTILS=On" if build.with? "utils"
     args << "-DGCC_INSTALL_PREFIX=#{Formula["gcc"].prefix}"