Add ThinLTO for LLVM >= 4.0 (#4367)

* Add ThinLTO for LLVM >= 4.0

Add --lto=thin as compiler option.
Emits .bc with summary metadata as .o
Replace linker with clang with lto_cache_dir
Forward n_threads compiler option to ThinLTO -mllvm,-threads=N
Turn off implicit --single-module on release build with ThinLTO

$ crystal build --lto=thin main.cr

NB: --no-debug is sometimes needed

* Allow compilation of LTO with LLVM >= 5.0

* Use clang executable by default

* Allow clang override via environment CLANG variable

* Split compile into compile to object and to lto prepared

src/compiler/crystal/command.cr

@@ -301,6 +301,12 @@ class Crystal::Command
         opts.on("--no-debug", "Skip any symbolic debug info") do
           compiler.debug = Crystal::Debug::None
         end
+        {% unless LibLLVM::IS_38 || LibLLVM::IS_39 %}
+        opts.on("--lto=FLAG", "Use ThinLTO --lto=thin") do |flag|
+          error "--lto=thin is the only lto supported option" unless flag == "thin"
+          compiler.thin_lto = true
+        end
+        {% end %}
       end
 
       opts.on("-D FLAG", "--define FLAG", "Define a compile-time flag") do |flag|

src/compiler/crystal/compiler.cr

@@ -126,6 +126,9 @@ module Crystal
     # Whether to link statically
     property? static = false
 
+    # Whether to use llvm ThinLTO for linking
+    property thin_lto = false
+
     # Compiles the given *source*, with *output_filename* as the name
     # of the generated executable.
     #
@@ -232,7 +235,7 @@ module Crystal
 
     private def codegen(program, node : ASTNode, sources, output_filename)
       llvm_modules = @progress_tracker.stage("Codegen (crystal)") do
-        program.codegen node, debug: debug, single_module: @single_module || @release || @cross_compile || @emit
+        program.codegen node, debug: debug, single_module: @single_module || (!@thin_lto && @release) || @cross_compile || @emit
       end
 
       if @cross_compile
@@ -283,10 +286,10 @@ module Crystal
 
       target_machine.emit_obj_to_file llvm_mod, object_name
 
-      stdout.puts linker_command(program, object_name, output_filename)
+      stdout.puts linker_command(program, object_name, output_filename, nil)
     end
 
-    private def linker_command(program : Program, object_name, output_filename)
+    private def linker_command(program : Program, object_name, output_filename, output_dir)
       if program.has_flag? "windows"
         if object_name
           object_name = %("#{object_name}")
@@ -300,6 +303,19 @@ module Crystal
 
         %(#{CL} #{object_name} "/Fe#{output_filename}" #{program.lib_flags} #{link_flags})
       else
+        if thin_lto
+          clang = ENV["CLANG"]? || "clang"
+          lto_cache_dir = "#{output_dir}/lto.cache"
+          Dir.mkdir_p(lto_cache_dir)
+          {% if flag?(:darwin) %}
+            cc = ENV["CC"]? || "#{clang} -flto=thin -Wl,-mllvm,-threads=#{n_threads},-cache_path_lto,#{lto_cache_dir},#{@release ? "-mllvm,-O2" : "-mllvm,-O0"}"
+          {% else %}
+            cc = ENV["CC"]? || "#{clang} -flto=thin -Wl,-plugin-opt,jobs=#{n_threads},-plugin-opt,cache-dir=#{lto_cache_dir} #{@release ? "-O2" : "-O0"}"
+          {% end %}
+        else
+          cc = CC
+        end
+
         if object_name
           object_name = %('#{object_name}')
         else
@@ -310,7 +326,7 @@ module Crystal
         link_flags += " -rdynamic"
         link_flags += " -static" if static?
 
-        %(#{CC} #{object_name} -o '#{output_filename}' #{link_flags} #{program.lib_flags})
+        %(#{cc} #{object_name} -o '#{output_filename}' #{link_flags} #{program.lib_flags})
       end
     end
 
@@ -345,7 +361,7 @@ module Crystal
 
       @progress_tracker.stage("Codegen (linking)") do
         Dir.cd(output_dir) do
-          system(linker_command(program, nil, output_filename), object_names)
+          system(linker_command(program, nil, output_filename, output_dir), object_names)
         end
       end
 
@@ -543,6 +559,24 @@ module Crystal
       end
 
       def compile
+        if compiler.thin_lto
+          compile_to_thin_lto
+        else
+          compile_to_object
+        end
+      end
+
+      private def compile_to_thin_lto
+        {% unless LibLLVM::IS_38 || LibLLVM::IS_39 %}
+          llvm_mod.write_bitcode_with_summary_to_file(object_name)
+          @reused_previous_compilation = false
+          dump_llvm_ir
+        {% else %}
+          raise {{ "ThinLTO is not available in LLVM #{LibLLVM::VERSION}".stringify }}
+        {% end %}
+      end
+
+      private def compile_to_object
         bc_name = self.bc_name
         object_name = self.object_name
 
@@ -592,6 +626,10 @@ module Crystal
           @reused_previous_compilation = true
         end
 
+        dump_llvm_ir
+      end
+
+      private def dump_llvm_ir
         llvm_mod.print_to_file ll_name if compiler.dump_ll?
       end
 

src/llvm/ext/llvm_ext.cc

@@ -6,6 +6,8 @@
 #include <llvm/IR/DebugLoc.h>
 #include <llvm/IR/LLVMContext.h>
 #include <llvm/IR/Metadata.h>
+#include <llvm/Support/raw_ostream.h>
+#include <llvm/Support/FileSystem.h>
 
 using namespace llvm;
 
@@ -18,6 +20,11 @@ using namespace llvm;
 #define LLVM_VERSION_LE(major, minor) \
   (LLVM_VERSION_MAJOR < (major) || LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR <= (minor))
 
+#if LLVM_VERSION_GE(4, 0)
+#include <llvm/Bitcode/BitcodeWriter.h>
+#include <llvm/Analysis/ModuleSummaryAnalysis.h>
+#endif
+
 #if LLVM_VERSION_LE(4, 0)
 typedef struct LLVMOpaqueDIBuilder *LLVMDIBuilderRef;
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DIBuilder, LLVMDIBuilderRef)
@@ -417,4 +424,18 @@ LLVMValueRef LLVMExtBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef
 #endif
 }
 
+void LLVMWriteBitcodeWithSummaryToFile(LLVMModuleRef mref, const char *File) {
+#if LLVM_VERSION_GE(4, 0)
+  // https://github.com/ldc-developers/ldc/pull/1840/files
+  Module *m = unwrap(mref);
+
+  std::error_code EC;
+  raw_fd_ostream OS(File, EC, sys::fs::F_None);
+  if (EC) return;
+
+  llvm::ModuleSummaryIndex moduleSummaryIndex = llvm::buildModuleSummaryIndex(*m, nullptr, nullptr);
+  llvm::WriteBitcodeToFile(m, OS, true, &moduleSummaryIndex, true);
+#endif
+}
+
 }

src/llvm/lib_llvm_ext.cr

@@ -134,4 +134,8 @@ lib LibLLVMExt
                                         then : LibLLVM::BasicBlockRef, catch : LibLLVM::BasicBlockRef,
                                         bundle : LibLLVMExt::OperandBundleDefRef,
                                         name : LibC::Char*) : LibLLVM::ValueRef
+
+  {% unless LibLLVM::IS_38 || LibLLVM::IS_39 %}
+    fun write_bitcode_with_summary_to_file = LLVMWriteBitcodeWithSummaryToFile(module : LibLLVM::ModuleRef, path : UInt8*) : Void
+  {% end %}
 end

src/llvm/module.cr

@@ -57,6 +57,12 @@ class LLVM::Module
     LibLLVM.write_bitcode_to_file self, filename
   end
 
+  {% unless LibLLVM::IS_38 || LibLLVM::IS_39 %}
+    def write_bitcode_with_summary_to_file(filename : String)
+      LibLLVMExt.write_bitcode_with_summary_to_file self, filename
+    end
+  {% end %}
+
   def write_bitcode_to_memory_buffer
     MemoryBuffer.new(LibLLVM.write_bitcode_to_memory_buffer self)
   end