Implement a file-path remapping feature in support of debuginfo and reproducible builds

src/doc/unstable-book/src/SUMMARY.md

@@ -2,6 +2,7 @@
 
 - [Compiler flags](compiler-flags.md)
     - [linker_flavor](compiler-flags/linker-flavor.md)
+    - [remap_path_prefix](compiler-flags/remap-path-prefix.md)
 - [Language features](language-features.md)
     - [abi_msp430_interrupt](language-features/abi-msp430-interrupt.md)
     - [abi_ptx](language-features/abi-ptx.md)

src/doc/unstable-book/src/compiler-flags/remap-path-prefix.md

@@ -0,0 +1,37 @@
+# `remap-path-prefix`
+
+The tracking issue for this feature is: [#41555](https://github.com/rust-lang/rust/issues/41555)
+
+------------------------
+
+The `-Z remap-path-prefix-from`, `-Z remap-path-prefix-to` commandline option
+pair allows to replace prefixes of any file paths the compiler emits in various
+places. This is useful for bringing debuginfo paths into a well-known form and
+for achieving reproducible builds independent of the directory the compiler was
+executed in. All paths emitted by the compiler are affected, including those in
+error messages.
+
+In order to map all paths starting with `/home/foo/my-project/src` to
+`/sources/my-project`, one would invoke the compiler as follows:
+
+```text
+rustc -Zremap-path-prefix-from="/home/foo/my-project/src" -Zremap-path-prefix-to="/sources/my-project"
+```
+
+Debuginfo for code from the file `/home/foo/my-project/src/foo/mod.rs`,
+for example, would then point debuggers to `/sources/my-project/foo/mod.rs`
+instead of the original file.
+
+The options can be specified multiple times when multiple prefixes should be
+mapped:
+
+```text
+rustc -Zremap-path-prefix-from="/home/foo/my-project/src" \
+      -Zremap-path-prefix-to="/sources/my-project" \
+      -Zremap-path-prefix-from="/home/foo/my-project/build-dir" \
+      -Zremap-path-prefix-to="/stable-build-dir"
+```
+
+When the options are given multiple times, the nth `-from` will be matched up
+with the nth `-to` and they can appear anywhere on the commandline. Mappings
+specified later on the line will take precedence over earlier ones.

src/grammar/verify.rs

@@ -296,7 +296,7 @@ fn main() {
                                          syntax::errors::registry::Registry::new(&[]),
                                          Rc::new(DummyCrateStore));
     let filemap = session.parse_sess.codemap()
-                         .new_filemap("<n/a>".to_string(), None, code);
+                         .new_filemap("<n/a>".to_string(), code);
     let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
     let cm = session.codemap();
 

src/librustc/session/config.rs

@@ -25,6 +25,7 @@ use lint;
 use middle::cstore;
 
 use syntax::ast::{self, IntTy, UintTy};
+use syntax::codemap::FilePathMapping;
 use syntax::parse::token;
 use syntax::parse;
 use syntax::symbol::Symbol;
@@ -492,6 +493,14 @@ impl Options {
         self.incremental.is_none() ||
         self.cg.codegen_units == 1
     }
+
+    pub fn file_path_mapping(&self) -> FilePathMapping {
+        FilePathMapping::new(
+            self.debugging_opts.remap_path_prefix_from.iter().zip(
+                self.debugging_opts.remap_path_prefix_to.iter()
+            ).map(|(src, dst)| (src.clone(), dst.clone())).collect()
+        )
+    }
 }
 
 // The type of entry function, so
@@ -1012,6 +1021,10 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "Set the optimization fuel quota for a crate."),
     print_fuel: Option<String> = (None, parse_opt_string, [TRACKED],
         "Make Rustc print the total optimization fuel used by a crate."),
+    remap_path_prefix_from: Vec<String> = (vec![], parse_string_push, [TRACKED],
+        "add a source pattern to the file path remapping config"),
+    remap_path_prefix_to: Vec<String> = (vec![], parse_string_push, [TRACKED],
+        "add a mapping target to the file path remapping config"),
 }
 
 pub fn default_lib_output() -> CrateType {
@@ -1319,7 +1332,7 @@ pub fn rustc_optgroups() -> Vec<RustcOptGroup> {
 // Convert strings provided as --cfg [cfgspec] into a crate_cfg
 pub fn parse_cfgspecs(cfgspecs: Vec<String> ) -> ast::CrateConfig {
     cfgspecs.into_iter().map(|s| {
-        let sess = parse::ParseSess::new();
+        let sess = parse::ParseSess::new(FilePathMapping::empty());
         let mut parser =
             parse::new_parser_from_source_str(&sess, "cfgspec".to_string(), s.to_string());
 
@@ -1430,6 +1443,23 @@ pub fn build_session_options_and_crate_config(matches: &getopts::Matches)
         output_types.insert(OutputType::Exe, None);
     }
 
+    let remap_path_prefix_sources = debugging_opts.remap_path_prefix_from.len();
+    let remap_path_prefix_targets = debugging_opts.remap_path_prefix_from.len();
+
+    if remap_path_prefix_targets < remap_path_prefix_sources {
+        for source in &debugging_opts.remap_path_prefix_from[remap_path_prefix_targets..] {
+            early_error(error_format,
+                &format!("option `-Zremap-path-prefix-from='{}'` does not have \
+                         a corresponding `-Zremap-path-prefix-to`", source))
+        }
+    } else if remap_path_prefix_targets > remap_path_prefix_sources {
+        for target in &debugging_opts.remap_path_prefix_to[remap_path_prefix_sources..] {
+            early_error(error_format,
+                &format!("option `-Zremap-path-prefix-to='{}'` does not have \
+                          a corresponding `-Zremap-path-prefix-from`", target))
+        }
+    }
+
     let mut cg = build_codegen_options(matches, error_format);
 
     // Issue #30063: if user requests llvm-related output to one

src/librustc/session/mod.rs

@@ -74,8 +74,10 @@ pub struct Session {
     // The name of the root source file of the crate, in the local file system.
     // The path is always expected to be absolute. `None` means that there is no
     // source file.
-    pub local_crate_source_file: Option<PathBuf>,
-    pub working_dir: PathBuf,
+    pub local_crate_source_file: Option<String>,
+    // The directory the compiler has been executed in plus a flag indicating
+    // if the value stored here has been affected by path remapping.
+    pub working_dir: (String, bool),
     pub lint_store: RefCell<lint::LintStore>,
     pub lints: RefCell<lint::LintTable>,
     /// Set of (LintId, span, message) tuples tracking lint (sub)diagnostics
@@ -553,12 +555,14 @@ pub fn build_session(sopts: config::Options,
                      registry: errors::registry::Registry,
                      cstore: Rc<CrateStore>)
                      -> Session {
+    let file_path_mapping = sopts.file_path_mapping();
+
     build_session_with_codemap(sopts,
                                dep_graph,
                                local_crate_source_file,
                                registry,
                                cstore,
-                               Rc::new(codemap::CodeMap::new()),
+                               Rc::new(codemap::CodeMap::new(file_path_mapping)),
                                None)
 }
 
@@ -622,7 +626,7 @@ pub fn build_session_(sopts: config::Options,
         Ok(t) => t,
         Err(e) => {
             panic!(span_diagnostic.fatal(&format!("Error loading host specification: {}", e)));
-    }
+        }
     };
     let target_cfg = config::build_target_config(&sopts, &span_diagnostic);
     let p_s = parse::ParseSess::with_span_handler(span_diagnostic, codemap);
@@ -631,21 +635,22 @@ pub fn build_session_(sopts: config::Options,
         None => Some(filesearch::get_or_default_sysroot())
     };
 
+    let file_path_mapping = sopts.file_path_mapping();
+
     // Make the path absolute, if necessary
-    let local_crate_source_file = local_crate_source_file.map(|path|
-        if path.is_absolute() {
-            path.clone()
-        } else {
-            env::current_dir().unwrap().join(&path)
-        }
-    );
+    let local_crate_source_file = local_crate_source_file.map(|path| {
+        file_path_mapping.map_prefix(path.to_string_lossy().into_owned()).0
+    });
 
     let optimization_fuel_crate = sopts.debugging_opts.fuel.as_ref().map(|i| i.0.clone());
     let optimization_fuel_limit = Cell::new(sopts.debugging_opts.fuel.as_ref()
         .map(|i| i.1).unwrap_or(0));
     let print_fuel_crate = sopts.debugging_opts.print_fuel.clone();
     let print_fuel = Cell::new(0);
 
+    let working_dir = env::current_dir().unwrap().to_string_lossy().into_owned();
+    let working_dir = file_path_mapping.map_prefix(working_dir);
+
     let sess = Session {
         dep_graph: dep_graph.clone(),
         target: target_cfg,
@@ -660,7 +665,7 @@ pub fn build_session_(sopts: config::Options,
         derive_registrar_fn: Cell::new(None),
         default_sysroot: default_sysroot,
         local_crate_source_file: local_crate_source_file,
-        working_dir: env::current_dir().unwrap(),
+        working_dir: working_dir,
         lint_store: RefCell::new(lint::LintStore::new()),
         lints: RefCell::new(lint::LintTable::new()),
         one_time_diagnostics: RefCell::new(FxHashSet()),

src/librustc_driver/lib.rs

@@ -206,7 +206,7 @@ pub fn run_compiler<'a>(args: &[String],
     let cstore = Rc::new(CStore::new(&dep_graph));
 
     let loader = file_loader.unwrap_or(box RealFileLoader);
-    let codemap = Rc::new(CodeMap::with_file_loader(loader));
+    let codemap = Rc::new(CodeMap::with_file_loader(loader, sopts.file_path_mapping()));
     let mut sess = session::build_session_with_codemap(
         sopts, &dep_graph, input_file_path, descriptions, cstore.clone(), codemap, emitter_dest,
     );

src/librustc_driver/test.rs

@@ -31,7 +31,7 @@ use rustc::session::{self, config};
 use std::rc::Rc;
 use syntax::ast;
 use syntax::abi::Abi;
-use syntax::codemap::CodeMap;
+use syntax::codemap::{CodeMap, FilePathMapping};
 use errors;
 use errors::emitter::Emitter;
 use errors::{Level, DiagnosticBuilder};
@@ -108,7 +108,7 @@ fn test_env<F>(source_string: &str,
                                        &dep_graph,
                                        None,
                                        diagnostic_handler,
-                                       Rc::new(CodeMap::new()),
+                                       Rc::new(CodeMap::new(FilePathMapping::empty())),
                                        cstore.clone());
     rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
     let input = config::Input::Str {

src/librustc_metadata/cstore_impl.rs

@@ -395,7 +395,7 @@ impl CrateStore for cstore::CStore {
         let (name, def) = data.get_macro(id.index);
         let source_name = format!("<{} macros>", name);
 
-        let filemap = sess.parse_sess.codemap().new_filemap(source_name, None, def.body);
+        let filemap = sess.parse_sess.codemap().new_filemap(source_name, def.body);
         let local_span = Span { lo: filemap.start_pos, hi: filemap.end_pos, ctxt: NO_EXPANSION };
         let body = filemap_to_stream(&sess.parse_sess, filemap);
 

src/librustc_metadata/decoder.rs

@@ -1134,7 +1134,7 @@ impl<'a, 'tcx> CrateMetadata {
                         // We can't reuse an existing FileMap, so allocate a new one
                         // containing the information we need.
                         let syntax_pos::FileMap { name,
-                                                  abs_path,
+                                                  name_was_remapped,
                                                   start_pos,
                                                   end_pos,
                                                   lines,
@@ -1158,7 +1158,7 @@ impl<'a, 'tcx> CrateMetadata {
                         }
 
                         let local_version = local_codemap.new_imported_filemap(name,
-                                                                               abs_path,
+                                                                               name_was_remapped,
                                                                                source_length,
                                                                                lines,
                                                                                multibyte_chars);

src/librustc_metadata/encoder.rs

@@ -30,6 +30,7 @@ use std::hash::Hash;
 use std::intrinsics;
 use std::io::prelude::*;
 use std::io::Cursor;
+use std::path::Path;
 use std::rc::Rc;
 use std::u32;
 use syntax::ast::{self, CRATE_NODE_ID};
@@ -1268,13 +1269,40 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     fn encode_codemap(&mut self) -> LazySeq<syntax_pos::FileMap> {
         let codemap = self.tcx.sess.codemap();
         let all_filemaps = codemap.files.borrow();
-        self.lazy_seq_ref(all_filemaps.iter()
+        let adapted = all_filemaps.iter()
             .filter(|filemap| {
                 // No need to re-export imported filemaps, as any downstream
                 // crate will import them from their original source.
                 !filemap.is_imported()
             })
-            .map(|filemap| &**filemap))
+            .map(|filemap| {
+                // When exporting FileMaps, we expand all paths to absolute
+                // paths because any relative paths are potentially relative to
+                // a wrong directory.
+                // However, if a path has been modified via
+                // `-Zremap-path-prefix` we assume the user has already set
+                // things up the way they want and don't touch the path values
+                // anymore.
+                let name = Path::new(&filemap.name);
+                let (ref working_dir, working_dir_was_remapped) = self.tcx.sess.working_dir;
+                if filemap.name_was_remapped ||
+                   (name.is_relative() && working_dir_was_remapped) {
+                    // This path of this FileMap has been modified by
+                    // path-remapping, so we use it verbatim (and avoid cloning
+                    // the whole map in the process).
+                    filemap.clone()
+                } else {
+                    let mut adapted = (**filemap).clone();
+                    let abs_path = Path::new(working_dir).join(name)
+                                                         .to_string_lossy()
+                                                         .into_owned();
+                    adapted.name = abs_path;
+                    Rc::new(adapted)
+                }
+            })
+            .collect::<Vec<_>>();
+
+        self.lazy_seq_ref(adapted.iter().map(|fm| &**fm))
     }
 
     fn encode_def_path_table(&mut self) -> Lazy<DefPathTable> {

src/librustc_save_analysis/dump_visitor.rs

@@ -37,6 +37,7 @@ use rustc::ty::{self, TyCtxt, AssociatedItemContainer};
 use std::collections::HashSet;
 use std::collections::hash_map::DefaultHasher;
 use std::hash::*;
+use std::path::Path;
 
 use syntax::ast::{self, NodeId, PatKind, Attribute, CRATE_NODE_ID};
 use syntax::parse::token;
@@ -128,6 +129,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
     pub fn dump_crate_info(&mut self, name: &str, krate: &ast::Crate) {
         let source_file = self.tcx.sess.local_crate_source_file.as_ref();
         let crate_root = source_file.map(|source_file| {
+            let source_file = Path::new(source_file);
             match source_file.file_name() {
                 Some(_) => source_file.parent().unwrap().display().to_string(),
                 None => source_file.display().to_string(),

src/librustc_trans/debuginfo/create_scope_map.rs

@@ -8,12 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use super::FunctionDebugContext;
+use super::{FunctionDebugContext, FunctionDebugContextData};
 use super::metadata::file_metadata;
 use super::utils::{DIB, span_start};
 
 use llvm;
-use llvm::debuginfo::{DIScope, DISubprogram};
+use llvm::debuginfo::DIScope;
 use common::CrateContext;
 use rustc::mir::{Mir, VisibilityScope};
 
@@ -53,8 +53,8 @@ pub fn create_mir_scopes(ccx: &CrateContext, mir: &Mir, debug_context: &Function
     };
     let mut scopes = IndexVec::from_elem(null_scope, &mir.visibility_scopes);
 
-    let fn_metadata = match *debug_context {
-        FunctionDebugContext::RegularContext(ref data) => data.fn_metadata,
+    let debug_context = match *debug_context {
+        FunctionDebugContext::RegularContext(ref data) => data,
         FunctionDebugContext::DebugInfoDisabled |
         FunctionDebugContext::FunctionWithoutDebugInfo => {
             return scopes;
@@ -71,7 +71,7 @@ pub fn create_mir_scopes(ccx: &CrateContext, mir: &Mir, debug_context: &Function
     // Instantiate all scopes.
     for idx in 0..mir.visibility_scopes.len() {
         let scope = VisibilityScope::new(idx);
-        make_mir_scope(ccx, &mir, &has_variables, fn_metadata, scope, &mut scopes);
+        make_mir_scope(ccx, &mir, &has_variables, debug_context, scope, &mut scopes);
     }
 
     scopes
@@ -80,7 +80,7 @@ pub fn create_mir_scopes(ccx: &CrateContext, mir: &Mir, debug_context: &Function
 fn make_mir_scope(ccx: &CrateContext,
                   mir: &Mir,
                   has_variables: &BitVector,
-                  fn_metadata: DISubprogram,
+                  debug_context: &FunctionDebugContextData,
                   scope: VisibilityScope,
                   scopes: &mut IndexVec<VisibilityScope, MirDebugScope>) {
     if scopes[scope].is_valid() {
@@ -89,13 +89,13 @@ fn make_mir_scope(ccx: &CrateContext,
 
     let scope_data = &mir.visibility_scopes[scope];
     let parent_scope = if let Some(parent) = scope_data.parent_scope {
-        make_mir_scope(ccx, mir, has_variables, fn_metadata, parent, scopes);
+        make_mir_scope(ccx, mir, has_variables, debug_context, parent, scopes);
         scopes[parent]
     } else {
         // The root is the function itself.
         let loc = span_start(ccx, mir.span);
         scopes[scope] = MirDebugScope {
-            scope_metadata: fn_metadata,
+            scope_metadata: debug_context.fn_metadata,
             file_start_pos: loc.file.start_pos,
             file_end_pos: loc.file.end_pos,
         };
@@ -109,14 +109,17 @@ fn make_mir_scope(ccx: &CrateContext,
         // However, we don't skip creating a nested scope if
         // our parent is the root, because we might want to
         // put arguments in the root and not have shadowing.
-        if parent_scope.scope_metadata != fn_metadata {
+        if parent_scope.scope_metadata != debug_context.fn_metadata {
             scopes[scope] = parent_scope;
             return;
         }
     }
 
     let loc = span_start(ccx, scope_data.span);
-    let file_metadata = file_metadata(ccx, &loc.file.name, &loc.file.abs_path);
+    let file_metadata = file_metadata(ccx,
+                                      &loc.file.name,
+                                      debug_context.defining_crate);
+
     let scope_metadata = unsafe {
         llvm::LLVMRustDIBuilderCreateLexicalBlock(
             DIB(ccx),