Preliminary work for incremental ThinLTO.

src/librustc/mir/mono.rs

@@ -8,15 +8,16 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use hir::def_id::DefId;
+use hir::def_id::{DefId, CrateNum};
 use syntax::ast::NodeId;
-use syntax::symbol::InternedString;
+use syntax::symbol::{Symbol, InternedString};
 use ty::{Instance, TyCtxt};
 use util::nodemap::FxHashMap;
 use rustc_data_structures::base_n;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasherResult,
                                            StableHasher};
 use ich::{Fingerprint, StableHashingContext, NodeIdHashingMode};
+use std::fmt;
 use std::hash::Hash;
 
 #[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)]
@@ -173,6 +174,80 @@ impl<'tcx> CodegenUnit<'tcx> {
             self.size_estimate = Some(size_estimate + delta);
         }
     }
+
+    /// CGU names should fulfill the following requirements:
+    /// - They should be able to act as a file name on any kind of file system
+    /// - They should not collide with other CGU names, even for different versions
+    ///   of the same crate.
+    ///
+    /// Consequently, we don't use special characters except for '.' and '-' and we
+    /// prefix each name with the crate-name and crate-disambiguator.
+    ///
+    /// This function will build CGU names of the form:
+    ///
+    /// ```
+    /// <crate-name>.<crate-disambiguator>(-<component>)*[.<special-suffix>]
+    /// ```
+    ///
+    /// The '.' before `<special-suffix>` makes sure that names with a special
+    /// suffix can never collide with a name built out of regular Rust
+    /// identifiers (e.g. module paths).
+    pub fn build_cgu_name<I, C, S>(tcx: TyCtxt,
+                                   cnum: CrateNum,
+                                   components: I,
+                                   special_suffix: Option<S>)
+                                   -> InternedString
+        where I: IntoIterator<Item=C>,
+              C: fmt::Display,
+              S: fmt::Display,
+    {
+        let cgu_name = CodegenUnit::build_cgu_name_no_mangle(tcx,
+                                                             cnum,
+                                                             components,
+                                                             special_suffix);
+
+        if tcx.sess.opts.debugging_opts.human_readable_cgu_names {
+            cgu_name
+        } else {
+            let cgu_name = &cgu_name.as_str()[..];
+            Symbol::intern(&CodegenUnit::mangle_name(cgu_name)).as_interned_str()
+        }
+    }
+
+    /// Same as `CodegenUnit::build_cgu_name()` but will never mangle the
+    /// resulting name.
+    pub fn build_cgu_name_no_mangle<I, C, S>(tcx: TyCtxt,
+                                             cnum: CrateNum,
+                                             components: I,
+                                             special_suffix: Option<S>)
+                                             -> InternedString
+        where I: IntoIterator<Item=C>,
+              C: fmt::Display,
+              S: fmt::Display,
+    {
+        use std::fmt::Write;
+
+        let mut cgu_name = String::with_capacity(64);
+
+        // Start out with the crate name and disambiguator
+        write!(cgu_name,
+               "{}.{}",
+               tcx.crate_name(cnum),
+               tcx.crate_disambiguator(cnum)).unwrap();
+
+        // Add the components
+        for component in components {
+            write!(cgu_name, "-{}", component).unwrap();
+        }
+
+        if let Some(special_suffix) = special_suffix {
+            // We add a dot in here so it cannot clash with anything in a regular
+            // Rust identifier
+            write!(cgu_name, ".{}", special_suffix).unwrap();
+        }
+
+        Symbol::intern(&cgu_name[..]).as_interned_str()
+    }
 }
 
 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for CodegenUnit<'tcx> {

src/librustc/session/mod.rs

@@ -26,6 +26,7 @@ use util::nodemap::{FxHashMap, FxHashSet};
 use util::common::{duration_to_secs_str, ErrorReported};
 use util::common::ProfileQueriesMsg;
 
+use rustc_data_structures::base_n;
 use rustc_data_structures::sync::{self, Lrc, Lock, LockCell, OneThread, Once, RwLock};
 
 use syntax::ast::NodeId;
@@ -1185,6 +1186,14 @@ impl CrateDisambiguator {
     }
 }
 
+impl fmt::Display for CrateDisambiguator {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        let (a, b) = self.0.as_value();
+        let as_u128 = a as u128 | ((b as u128) << 64);
+        f.write_str(&base_n::encode(as_u128, base_n::CASE_INSENSITIVE))
+    }
+}
+
 impl From<Fingerprint> for CrateDisambiguator {
     fn from(fingerprint: Fingerprint) -> CrateDisambiguator {
         CrateDisambiguator(fingerprint)

src/librustc_codegen_llvm/back/lto.rs

@@ -20,16 +20,23 @@ use rustc::hir::def_id::LOCAL_CRATE;
 use rustc::middle::exported_symbols::SymbolExportLevel;
 use rustc::session::config::{self, Lto};
 use rustc::util::common::time_ext;
+use rustc_data_structures::fx::FxHashMap;
 use time_graph::Timeline;
 use {ModuleCodegen, ModuleLlvm, ModuleKind, ModuleSource};
 
 use libc;
 
-use std::ffi::CString;
+use std::ffi::{CString, CStr};
+use std::fs::File;
+use std::io;
+use std::mem;
+use std::path::Path;
 use std::ptr;
 use std::slice;
 use std::sync::Arc;
 
+pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-imports.bin";
+
 pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool {
     match crate_type {
         config::CrateTypeExecutable |
@@ -193,7 +200,7 @@ pub(crate) fn run(cgcx: &CodegenContext,
         }
         Lto::Thin |
         Lto::ThinLocal => {
-            thin_lto(&diag_handler, modules, upstream_modules, &arr, timeline)
+            thin_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline)
         }
         Lto::No => unreachable!(),
     }
@@ -346,7 +353,8 @@ impl Drop for Linker {
 /// calculating the *index* for ThinLTO. This index will then be shared amongst
 /// all of the `LtoModuleCodegen` units returned below and destroyed once
 /// they all go out of scope.
-fn thin_lto(diag_handler: &Handler,
+fn thin_lto(cgcx: &CodegenContext,
+            diag_handler: &Handler,
             modules: Vec<ModuleCodegen>,
             serialized_modules: Vec<(SerializedModule, CString)>,
             symbol_white_list: &[*const libc::c_char],
@@ -424,6 +432,18 @@ fn thin_lto(diag_handler: &Handler,
             let msg = format!("failed to prepare thin LTO context");
             return Err(write::llvm_err(&diag_handler, msg))
         }
+
+        // Save the ThinLTO import information for incremental compilation.
+        if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir {
+            let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME);
+            let imports = ThinLTOImports::from_thin_lto_data(data);
+            if let Err(err) = imports.save_to_file(&path) {
+                let msg = format!("Error while writing ThinLTO import data: {}",
+                                  err);
+                return Err(write::llvm_err(&diag_handler, msg));
+            }
+        }
+
         let data = ThinData(data);
         info!("thin LTO data created");
         timeline.record("data");
@@ -776,3 +796,129 @@ impl ThinModule {
         Ok(module)
     }
 }
+
+
+#[derive(Debug)]
+pub struct ThinLTOImports {
+    // key = llvm name of importing module, value = list of modules it imports from
+    imports: FxHashMap<String, Vec<String>>,
+}
+
+impl ThinLTOImports {
+
+    pub fn new_empty() -> ThinLTOImports {
+        ThinLTOImports {
+            imports: FxHashMap(),
+        }
+    }
+
+    /// Load the ThinLTO import map from ThinLTOData.
+    unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports {
+        let raw_data: *const llvm::ThinLTOModuleImports =
+            llvm::LLVMRustGetThinLTOModuleImports(data);
+
+        assert!(!raw_data.is_null());
+
+        let mut imports = FxHashMap();
+        let mut module_ptr = raw_data;
+        let mut module_index = 0;
+
+        loop {
+            let mut entry_ptr: *const llvm::ThinLTOModuleName = *module_ptr;
+
+            if entry_ptr.is_null() {
+                break;
+            }
+
+            let importing_module_name = CStr::from_ptr(*entry_ptr)
+                .to_str()
+                .expect("Non-utf8 LLVM module name encountered")
+                .to_owned();
+
+            entry_ptr = entry_ptr.offset(1);
+
+            let mut imported_modules = vec![];
+
+            loop {
+                let imported_module_name = *entry_ptr;
+
+                if imported_module_name.is_null() {
+                    break
+                }
+
+                let imported_module_name = CStr::from_ptr(imported_module_name)
+                    .to_str()
+                    .expect("Non-utf8 LLVM module name encountered")
+                    .to_owned();
+
+                imported_modules.push(imported_module_name);
+                entry_ptr = entry_ptr.offset(1);
+            }
+
+            imports.insert(importing_module_name, imported_modules);
+
+            module_ptr = module_ptr.offset(1);
+            module_index += 1;
+        }
+
+        assert_eq!(module_index, imports.len());
+
+        ThinLTOImports {
+            imports
+        }
+    }
+
+    pub fn save_to_file(&self, path: &Path) -> io::Result<()> {
+        use std::io::Write;
+
+        let file = File::create(path)?;
+        let mut writer = io::BufWriter::new(file);
+
+        for (importing_module_name, imported_modules) in &self.imports {
+            writeln!(writer, "{}", importing_module_name)?;
+
+            for imported_module in imported_modules {
+                writeln!(writer, "  {}", imported_module)?;
+            }
+
+            writeln!(writer)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn load_from_file(path: &Path) -> io::Result<ThinLTOImports> {
+        use std::io::BufRead;
+
+        let mut imports = FxHashMap();
+        let mut current_module = None;
+        let mut current_imports = vec![];
+
+        let file = File::open(path)?;
+
+        for line in io::BufReader::new(file).lines() {
+            let line = line?;
+
+            if line.is_empty() {
+                let importing_module = current_module
+                    .take()
+                    .expect("Importing module not set");
+
+                imports.insert(importing_module,
+                               mem::replace(&mut current_imports, vec![]));
+            } else if line.starts_with(" ") {
+                // This is an imported module
+                assert_ne!(current_module, None);
+                current_imports.push(line.trim().to_string());
+            } else {
+                // This is the beginning of a new module
+                assert_eq!(current_module, None);
+                current_module = Some(line.trim().to_string());
+            }
+        }
+
+        Ok(ThinLTOImports {
+            imports
+        })
+    }
+}

src/librustc_codegen_llvm/base.rs

@@ -29,7 +29,7 @@ use super::ModuleCodegen;
 use super::ModuleKind;
 
 use abi;
-use back::link;
+use back::{link, lto};
 use back::write::{self, OngoingCodegen, create_target_machine};
 use llvm::{ContextRef, ModuleRef, ValueRef, Vector, get_param};
 use llvm;
@@ -1370,6 +1370,27 @@ mod temp_stable_hash_impls {
     }
 }
 
+#[allow(unused)]
+fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports {
+    let path = rustc_incremental::in_incr_comp_dir_sess(
+        sess,
+        lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME
+    );
+
+    if !path.exists() {
+        return lto::ThinLTOImports::new_empty();
+    }
+
+    match lto::ThinLTOImports::load_from_file(&path) {
+        Ok(imports) => imports,
+        Err(e) => {
+            let msg = format!("Error while trying to load ThinLTO import data \
+                               for incremental compilation: {}", e);
+            sess.fatal(&msg)
+        }
+    }
+}
+
 pub fn define_custom_section(cx: &CodegenCx, def_id: DefId) {
     use rustc::mir::interpret::GlobalId;
 
@@ -1408,3 +1429,4 @@ pub fn define_custom_section(cx: &CodegenCx, def_id: DefId) {
         );
     }
 }
+

src/librustc_codegen_llvm/lib.rs

@@ -89,7 +89,7 @@ mod back {
     mod command;
     pub mod linker;
     pub mod link;
-    mod lto;
+    pub mod lto;
     pub mod symbol_export;
     pub mod write;
     mod rpath;