diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 76a6ffbb1c5b2..e3d69fc5c76df 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -584,6 +584,20 @@ pub(crate) fn run_pass_manager( // tools/lto/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { + if write::should_use_new_llvm_pass_manager(config) { + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + // See comment below for why this is necessary. + let opt_level = if let config::OptLevel::No = opt_level { + config::OptLevel::Less + } else { + opt_level + }; + write::optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage); + debug!("lto done"); + return; + } + let pm = llvm::LLVMCreatePassManager(); llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm); diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index 7dd57da90c3d6..9008970847a59 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -111,6 +111,18 @@ pub fn to_llvm_opt_settings( } } +fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel { + use config::OptLevel::*; + match cfg { + No => llvm::PassBuilderOptLevel::O0, + Less => llvm::PassBuilderOptLevel::O1, + Default => llvm::PassBuilderOptLevel::O2, + Aggressive => llvm::PassBuilderOptLevel::O3, + Size => llvm::PassBuilderOptLevel::Os, + SizeMin => llvm::PassBuilderOptLevel::Oz, + } +} + // If find_features is true this won't access `sess.crate_types` by assuming // that `is_pie_binary` is false. When we discover LLVM target features // `sess.crate_types` is uninitialized so we cannot access it. @@ -303,6 +315,88 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void } } +fn get_pgo_gen_path(config: &ModuleConfig) -> Option { + match config.pgo_gen { + SwitchWithOptPath::Enabled(ref opt_dir_path) => { + let path = if let Some(dir_path) = opt_dir_path { + dir_path.join("default_%m.profraw") + } else { + PathBuf::from("default_%m.profraw") + }; + + Some(CString::new(format!("{}", path.display())).unwrap()) + } + SwitchWithOptPath::Disabled => None, + } +} + +fn get_pgo_use_path(config: &ModuleConfig) -> Option { + config + .pgo_use + .as_ref() + .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) +} + +pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool { + // We only support the new pass manager starting with LLVM 9. + if llvm_util::get_major_version() < 9 { + return false; + } + + // The new pass manager is disabled by default. + config.new_llvm_pass_manager.unwrap_or(false) +} + +pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( + module: &ModuleCodegen, + config: &ModuleConfig, + opt_level: config::OptLevel, + opt_stage: llvm::OptStage, +) { + let unroll_loops = + opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin; + let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); + let pgo_gen_path = get_pgo_gen_path(config); + let pgo_use_path = get_pgo_use_path(config); + let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + // Sanitizer instrumentation is only inserted during the pre-link optimization stage. + let sanitizer_options = if !is_lto { + config.sanitizer.as_ref().map(|s| llvm::SanitizerOptions { + sanitize_memory: *s == Sanitizer::Memory, + sanitize_thread: *s == Sanitizer::Thread, + sanitize_address: *s == Sanitizer::Address, + sanitize_recover: config.sanitizer_recover.contains(s), + sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, + }) + } else { + None + }; + + // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. + // We would have to add upstream support for this first, before we can support + // config.inline_threshold and our more aggressive default thresholds. + // FIXME: NewPM uses an different and more explicit way to textually represent + // pass pipelines. It would probably make sense to expose this, but it would + // require a different format than the current -C passes. + llvm::LLVMRustOptimizeWithNewPassManager( + module.module_llvm.llmod(), + &*module.module_llvm.tm, + to_pass_builder_opt_level(opt_level), + opt_stage, + config.no_prepopulate_passes, + config.verify_llvm_ir, + using_thin_buffers, + config.merge_functions, + unroll_loops, + config.vectorize_slp, + config.vectorize_loop, + config.no_builtins, + sanitizer_options.as_ref(), + pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + ); +} + // Unsafe due to LLVM calls. pub(crate) unsafe fn optimize( cgcx: &CodegenContext, @@ -327,6 +421,17 @@ pub(crate) unsafe fn optimize( } if let Some(opt_level) = config.opt_level { + if should_use_new_llvm_pass_manager(config) { + let opt_stage = match cgcx.lto { + Lto::Fat => llvm::OptStage::PreLinkFatLTO, + Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, + _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, + _ => llvm::OptStage::PreLinkNoLTO, + }; + optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage); + return Ok(()); + } + // Create the two optimizing pass managers. These mirror what clang // does, and are by populated by LLVM's default PassManagerBuilder. // Each manager has a different set of passes, but they also share @@ -757,24 +862,8 @@ pub unsafe fn with_llvm_pmb( let opt_size = config.opt_size.map(|x| to_llvm_opt_settings(x).1).unwrap_or(llvm::CodeGenOptSizeNone); let inline_threshold = config.inline_threshold; - - let pgo_gen_path = match config.pgo_gen { - SwitchWithOptPath::Enabled(ref opt_dir_path) => { - let path = if let Some(dir_path) = opt_dir_path { - dir_path.join("default_%m.profraw") - } else { - PathBuf::from("default_%m.profraw") - }; - - Some(CString::new(format!("{}", path.display())).unwrap()) - } - SwitchWithOptPath::Disabled => None, - }; - - let pgo_use_path = config - .pgo_use - .as_ref() - .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()); + let pgo_gen_path = get_pgo_gen_path(config); + let pgo_use_path = get_pgo_use_path(config); llvm::LLVMRustConfigurePassManagerBuilder( builder, diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs index cdb55b179a3a2..f570e808f56f4 100644 --- a/src/librustc_codegen_llvm/llvm/ffi.rs +++ b/src/librustc_codegen_llvm/llvm/ffi.rs @@ -402,6 +402,38 @@ pub enum CodeGenOptLevel { Aggressive, } +/// LLVMRustPassBuilderOptLevel +#[repr(C)] +pub enum PassBuilderOptLevel { + O0, + O1, + O2, + O3, + Os, + Oz, +} + +/// LLVMRustOptStage +#[derive(PartialEq)] +#[repr(C)] +pub enum OptStage { + PreLinkNoLTO, + PreLinkThinLTO, + PreLinkFatLTO, + ThinLTO, + FatLTO, +} + +/// LLVMRustSanitizerOptions +#[repr(C)] +pub struct SanitizerOptions { + pub sanitize_memory: bool, + pub sanitize_thread: bool, + pub sanitize_address: bool, + pub sanitize_recover: bool, + pub sanitize_memory_track_origins: c_int, +} + /// LLVMRelocMode #[derive(Copy, Clone, PartialEq)] #[repr(C)] @@ -1896,6 +1928,23 @@ extern "C" { Output: *const c_char, FileType: FileType, ) -> LLVMRustResult; + pub fn LLVMRustOptimizeWithNewPassManager( + M: &'a Module, + TM: &'a TargetMachine, + OptLevel: PassBuilderOptLevel, + OptStage: OptStage, + NoPrepopulatePasses: bool, + VerifyIR: bool, + UseThinLTOBuffers: bool, + MergeFunctions: bool, + UnrollLoops: bool, + SLPVectorize: bool, + LoopVectorize: bool, + DisableSimplifyLibCalls: bool, + SanitizerOptions: Option<&SanitizerOptions>, + PGOGenPath: *const c_char, + PGOUsePath: *const c_char, + ); pub fn LLVMRustPrintModule( M: &'a Module, Output: *const c_char, diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs index 9905b3a56c0fa..92f795acc5438 100644 --- a/src/librustc_codegen_ssa/back/write.rs +++ b/src/librustc_codegen_ssa/back/write.rs @@ -88,6 +88,7 @@ pub struct ModuleConfig { pub vectorize_slp: bool, pub merge_functions: bool, pub inline_threshold: Option, + pub new_llvm_pass_manager: Option, // Instead of creating an object file by doing LLVM codegen, just // make the object file bitcode. Provides easy compatibility with // emscripten's ecc compiler, when used as the linker. @@ -132,6 +133,7 @@ impl ModuleConfig { vectorize_slp: false, merge_functions: false, inline_threshold: None, + new_llvm_pass_manager: None, } } @@ -140,6 +142,7 @@ impl ModuleConfig { self.no_prepopulate_passes = sess.opts.cg.no_prepopulate_passes; self.no_builtins = no_builtins || sess.target.target.options.no_builtins; self.inline_threshold = sess.opts.cg.inline_threshold; + self.new_llvm_pass_manager = sess.opts.debugging_opts.new_llvm_pass_manager; self.obj_is_bitcode = sess.target.target.options.obj_is_bitcode || sess.opts.cg.linker_plugin_lto.enabled(); let embed_bitcode = diff --git a/src/librustc_session/options.rs b/src/librustc_session/options.rs index 4b3645cce723a..a794670d7b8fe 100644 --- a/src/librustc_session/options.rs +++ b/src/librustc_session/options.rs @@ -968,4 +968,6 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "compile without linking"), link_only: bool = (false, parse_bool, [TRACKED], "link the `.rlink` file generated by `-Z no-link`"), + new_llvm_pass_manager: Option = (None, parse_opt_bool, [TRACKED], + "use new LLVM pass manager"), } diff --git a/src/rustllvm/PassWrapper.cpp b/src/rustllvm/PassWrapper.cpp index 4ac7e0e6e1fc0..15e2251d76321 100644 --- a/src/rustllvm/PassWrapper.cpp +++ b/src/rustllvm/PassWrapper.cpp @@ -12,6 +12,11 @@ #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/AssemblyAnnotationWriter.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Passes/PassBuilder.h" +#if LLVM_VERSION_GE(9, 0) +#include "llvm/Passes/StandardInstrumentations.h" +#endif #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Host.h" @@ -32,9 +37,12 @@ #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" #endif +#if LLVM_VERSION_GE(9, 0) +#include "llvm/Transforms/Utils/CanonicalizeAliases.h" +#endif +#include "llvm/Transforms/Utils/NameAnonGlobals.h" using namespace llvm; -using namespace llvm::legacy; typedef struct LLVMOpaquePass *LLVMPassRef; typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; @@ -314,6 +322,34 @@ static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { } } +enum class LLVMRustPassBuilderOptLevel { + O0, + O1, + O2, + O3, + Os, + Oz, +}; + +static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { + switch (Level) { + case LLVMRustPassBuilderOptLevel::O0: + return PassBuilder::O0; + case LLVMRustPassBuilderOptLevel::O1: + return PassBuilder::O1; + case LLVMRustPassBuilderOptLevel::O2: + return PassBuilder::O2; + case LLVMRustPassBuilderOptLevel::O3: + return PassBuilder::O3; + case LLVMRustPassBuilderOptLevel::Os: + return PassBuilder::Os; + case LLVMRustPassBuilderOptLevel::Oz: + return PassBuilder::Oz; + default: + report_fatal_error("Bad PassBuilderOptLevel."); + } +} + enum class LLVMRustRelocMode { Default, Static, @@ -604,6 +640,212 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, return LLVMRustResult::Success; } +enum class LLVMRustOptStage { + PreLinkNoLTO, + PreLinkThinLTO, + PreLinkFatLTO, + ThinLTO, + FatLTO, +}; + +struct LLVMRustSanitizerOptions { + bool SanitizeMemory; + bool SanitizeThread; + bool SanitizeAddress; + bool SanitizeRecover; + int SanitizeMemoryTrackOrigins; +}; + +extern "C" void +LLVMRustOptimizeWithNewPassManager( + LLVMModuleRef ModuleRef, + LLVMTargetMachineRef TMRef, + LLVMRustPassBuilderOptLevel OptLevelRust, + LLVMRustOptStage OptStage, + bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, + bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, + bool DisableSimplifyLibCalls, + LLVMRustSanitizerOptions *SanitizerOptions, + const char *PGOGenPath, const char *PGOUsePath) { +#if LLVM_VERSION_GE(9, 0) + Module *TheModule = unwrap(ModuleRef); + TargetMachine *TM = unwrap(TMRef); + PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust); + + // FIXME: MergeFunctions is not supported by NewPM yet. + (void) MergeFunctions; + + PipelineTuningOptions PTO; + PTO.LoopUnrolling = UnrollLoops; + PTO.LoopInterleaving = UnrollLoops; + PTO.LoopVectorization = LoopVectorize; + PTO.SLPVectorization = SLPVectorize; + + PassInstrumentationCallbacks PIC; + StandardInstrumentations SI; + SI.registerCallbacks(PIC); + + Optional PGOOpt; + if (PGOGenPath) { + assert(!PGOUsePath); + PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr); + } else if (PGOUsePath) { + assert(!PGOGenPath); + PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse); + } + + PassBuilder PB(TM, PTO, PGOOpt, &PIC); + + // FIXME: We may want to expose this as an option. + bool DebugPassManager = false; + LoopAnalysisManager LAM(DebugPassManager); + FunctionAnalysisManager FAM(DebugPassManager); + CGSCCAnalysisManager CGAM(DebugPassManager); + ModuleAnalysisManager MAM(DebugPassManager); + + FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + + Triple TargetTriple(TheModule->getTargetTriple()); + std::unique_ptr TLII(new TargetLibraryInfoImpl(TargetTriple)); + if (DisableSimplifyLibCalls) + TLII->disableAllFunctions(); + FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); + + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + // We manually collect pipeline callbacks so we can apply them at O0, where the + // PassBuilder does not create a pipeline. + std::vector> PipelineStartEPCallbacks; + std::vector> + OptimizerLastEPCallbacks; + + if (VerifyIR) { + PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) { + MPM.addPass(VerifierPass()); + }); + } + + if (SanitizerOptions) { + if (SanitizerOptions->SanitizeMemory) { + MemorySanitizerOptions Options( + SanitizerOptions->SanitizeMemoryTrackOrigins, + SanitizerOptions->SanitizeRecover, + /*CompileKernel=*/false); +#if LLVM_VERSION_GE(10, 0) + PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) { + MPM.addPass(MemorySanitizerPass(Options)); + }); +#endif + OptimizerLastEPCallbacks.push_back( + [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { + FPM.addPass(MemorySanitizerPass(Options)); + } + ); + } + + if (SanitizerOptions->SanitizeThread) { +#if LLVM_VERSION_GE(10, 0) + PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) { + MPM.addPass(ThreadSanitizerPass()); + }); +#endif + OptimizerLastEPCallbacks.push_back( + [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { + FPM.addPass(ThreadSanitizerPass()); + } + ); + } + + if (SanitizerOptions->SanitizeAddress) { + // FIXME: Rust does not expose the UseAfterScope option. + PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) { + MPM.addPass(RequireAnalysisPass()); + }); + OptimizerLastEPCallbacks.push_back( + [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { + FPM.addPass(AddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover)); + } + ); + PipelineStartEPCallbacks.push_back( + [SanitizerOptions](ModulePassManager &MPM) { + MPM.addPass(ModuleAddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover)); + } + ); + } + } + + ModulePassManager MPM(DebugPassManager); + if (!NoPrepopulatePasses) { + if (OptLevel == PassBuilder::O0) { + for (const auto &C : PipelineStartEPCallbacks) + C(MPM); + + if (!OptimizerLastEPCallbacks.empty()) { + FunctionPassManager FPM(DebugPassManager); + for (const auto &C : OptimizerLastEPCallbacks) + C(FPM, OptLevel); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + } + + MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false)); + +#if LLVM_VERSION_GE(10, 0) + if (PGOOpt) { + PB.addPGOInstrPassesForO0( + MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr, + /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); + } +#endif + } else { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + + switch (OptStage) { + case LLVMRustOptStage::PreLinkNoLTO: + MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); + break; + case LLVMRustOptStage::PreLinkThinLTO: + MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); + break; + case LLVMRustOptStage::PreLinkFatLTO: + MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); + break; + case LLVMRustOptStage::ThinLTO: + // FIXME: Does it make sense to pass the ModuleSummaryIndex? + // It only seems to be needed for C++ specific optimizations. + MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); + break; + case LLVMRustOptStage::FatLTO: + MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); + break; + } + } + } + + if (UseThinLTOBuffers) { + MPM.addPass(CanonicalizeAliasesPass()); + MPM.addPass(NameAnonGlobalPass()); + } + + // Upgrade all calls to old intrinsics first. + for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) + UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + + MPM.run(*TheModule, MAM); +#else + // The new pass manager has been available for a long time, + // but we don't bother supporting it on old LLVM versions. + report_fatal_error("New pass manager only supported since LLVM 9"); +#endif +} // Callback to demangle function name // Parameters: diff --git a/src/test/codegen/sanitizer-memory-track-orgins.rs b/src/test/codegen/sanitizer-memory-track-orgins.rs index 1fd496b35dfcc..8ea41c5d44bb1 100644 --- a/src/test/codegen/sanitizer-memory-track-orgins.rs +++ b/src/test/codegen/sanitizer-memory-track-orgins.rs @@ -15,10 +15,10 @@ #![crate_type="lib"] // MSAN-0-NOT: @__msan_track_origins -// MSAN-1: @__msan_track_origins = weak_odr local_unnamed_addr constant i32 1 -// MSAN-2: @__msan_track_origins = weak_odr local_unnamed_addr constant i32 2 -// MSAN-1-LTO: @__msan_track_origins = weak_odr local_unnamed_addr constant i32 1 -// MSAN-2-LTO: @__msan_track_origins = weak_odr local_unnamed_addr constant i32 2 +// MSAN-1: @__msan_track_origins = weak_odr {{.*}}constant i32 1 +// MSAN-2: @__msan_track_origins = weak_odr {{.*}}constant i32 2 +// MSAN-1-LTO: @__msan_track_origins = weak_odr {{.*}}constant i32 1 +// MSAN-2-LTO: @__msan_track_origins = weak_odr {{.*}}constant i32 2 // // MSAN-0-LABEL: define void @copy( // MSAN-1-LABEL: define void @copy(