From ffa4d7e87f7565177693eefb4650bb32c3498968 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Wed, 7 Aug 2019 09:30:16 -0700 Subject: [PATCH 1/6] Sort the fat LTO modules to produce deterministic output. --- src/librustc_codegen_ssa/back/write.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs index c9e4663fdbddf..2bd46b9f6efee 100644 --- a/src/librustc_codegen_ssa/back/write.rs +++ b/src/librustc_codegen_ssa/back/write.rs @@ -755,6 +755,15 @@ pub enum FatLTOInput { InMemory(ModuleCodegen), } +impl FatLTOInput { + fn name(&'a self) -> &'a String { + match self { + FatLTOInput::Serialized { name, buffer: _ } => &name, + FatLTOInput::InMemory(module) => &module.name, + } + } +} + fn execute_work_item( cgcx: &CodegenContext, work_item: WorkItem, @@ -1345,10 +1354,15 @@ fn start_executing_work( assert!(!started_lto); started_lto = true; - let needs_fat_lto = mem::take(&mut needs_fat_lto); + let mut needs_fat_lto: Vec> = mem::take(&mut needs_fat_lto); let needs_thin_lto = mem::take(&mut needs_thin_lto); let import_only_modules = mem::take(&mut lto_import_only_modules); + // Regardless of what order these modules completed in, report them to + // the backend in the same order every time to ensure that we're handing + // out deterministic results. + needs_fat_lto.sort_by(|m1, m2| m1.name().cmp(m2.name())); + for (work, cost) in generate_lto_work(&cgcx, needs_fat_lto, needs_thin_lto, import_only_modules) { let insertion_index = work_items From 5b2c5e181ac321f04621ef0b7dc78354bf3397d3 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Thu, 8 Aug 2019 07:54:27 -0700 Subject: [PATCH 2/6] Sort fat LTO modules later and add a test. --- src/librustc_codegen_llvm/back/lto.rs | 6 ++++-- src/librustc_codegen_ssa/back/write.rs | 16 +--------------- .../reproducible-build/Makefile | 11 ++++++++++- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 5d3cc0c0a255f..33b4c8eec54ce 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -265,7 +265,7 @@ fn fat_lto(cgcx: &CodegenContext, // and we want to move everything to the same LLVM context. Currently the // way we know of to do that is to serialize them to a string and them parse // them later. Not great but hey, that's why it's "fat" LTO, right? - serialized_modules.extend(modules.into_iter().map(|module| { + let mut new_modules = modules.into_iter().map(|module| { match module { FatLTOInput::InMemory(module) => { let buffer = ModuleBuffer::new(module.module_llvm.llmod()); @@ -277,7 +277,9 @@ fn fat_lto(cgcx: &CodegenContext, (SerializedModule::Local(buffer), llmod_id) } } - })); + }).collect::>(); + new_modules.sort_by(|module1, module2| module1.1.partial_cmp(&module2.1).unwrap()); + serialized_modules.extend(new_modules); serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| { (buffer, CString::new(wp.cgu_name).unwrap()) })); diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs index 2bd46b9f6efee..c9e4663fdbddf 100644 --- a/src/librustc_codegen_ssa/back/write.rs +++ b/src/librustc_codegen_ssa/back/write.rs @@ -755,15 +755,6 @@ pub enum FatLTOInput { InMemory(ModuleCodegen), } -impl FatLTOInput { - fn name(&'a self) -> &'a String { - match self { - FatLTOInput::Serialized { name, buffer: _ } => &name, - FatLTOInput::InMemory(module) => &module.name, - } - } -} - fn execute_work_item( cgcx: &CodegenContext, work_item: WorkItem, @@ -1354,15 +1345,10 @@ fn start_executing_work( assert!(!started_lto); started_lto = true; - let mut needs_fat_lto: Vec> = mem::take(&mut needs_fat_lto); + let needs_fat_lto = mem::take(&mut needs_fat_lto); let needs_thin_lto = mem::take(&mut needs_thin_lto); let import_only_modules = mem::take(&mut lto_import_only_modules); - // Regardless of what order these modules completed in, report them to - // the backend in the same order every time to ensure that we're handing - // out deterministic results. - needs_fat_lto.sort_by(|m1, m2| m1.name().cmp(m2.name())); - for (work, cost) in generate_lto_work(&cgcx, needs_fat_lto, needs_thin_lto, import_only_modules) { let insertion_index = work_items diff --git a/src/test/run-make-fulldeps/reproducible-build/Makefile b/src/test/run-make-fulldeps/reproducible-build/Makefile index a17ec212cfd58..5b9c9d3d03521 100644 --- a/src/test/run-make-fulldeps/reproducible-build/Makefile +++ b/src/test/run-make-fulldeps/reproducible-build/Makefile @@ -10,7 +10,8 @@ all: \ link_paths \ remap_paths \ different_source_dirs \ - extern_flags + extern_flags \ + fat_lto smoke: rm -rf $(TMPDIR) && mkdir $(TMPDIR) @@ -76,3 +77,11 @@ extern_flags: --extern reproducible_build_aux=$(TMPDIR)/libbar.rlib \ --crate-type rlib cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1 + +fat_lto: + rm -rf $(TMPDIR) && mkdir $(TMPDIR) + $(RUSTC) reproducible-build-aux.rs + $(RUSTC) reproducible-build.rs -C lto=fat + cp $(TMPDIR)/reproducible-build $(TMPDIR)/reproducible-build-a + $(RUSTC) reproducible-build.rs -C lto=fat + cmp "$(TMPDIR)/reproducible-build-a" "$(TMPDIR)/reproducible-build" || exit 1 From a46e36f9df68f9c59157ccc3c1519413802993b5 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Thu, 8 Aug 2019 09:05:32 -0700 Subject: [PATCH 3/6] Fix fat LTO determinism test so it fails without the fix. --- src/test/run-make-fulldeps/reproducible-build/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/run-make-fulldeps/reproducible-build/Makefile b/src/test/run-make-fulldeps/reproducible-build/Makefile index 5b9c9d3d03521..addbf9928bf4a 100644 --- a/src/test/run-make-fulldeps/reproducible-build/Makefile +++ b/src/test/run-make-fulldeps/reproducible-build/Makefile @@ -81,7 +81,7 @@ extern_flags: fat_lto: rm -rf $(TMPDIR) && mkdir $(TMPDIR) $(RUSTC) reproducible-build-aux.rs - $(RUSTC) reproducible-build.rs -C lto=fat + $(RUSTC) reproducible-build.rs -C lto=fat -C opt-level=1 cp $(TMPDIR)/reproducible-build $(TMPDIR)/reproducible-build-a - $(RUSTC) reproducible-build.rs -C lto=fat + $(RUSTC) reproducible-build.rs -C lto=fat -C opt-level=1 cmp "$(TMPDIR)/reproducible-build-a" "$(TMPDIR)/reproducible-build" || exit 1 From 3e6a9273057aec611b9f22025bb28355060af265 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Thu, 8 Aug 2019 10:51:52 -0700 Subject: [PATCH 4/6] Explain why we're sorting the modules. --- src/librustc_codegen_llvm/back/lto.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 33b4c8eec54ce..5ed08943fe6fd 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -278,6 +278,7 @@ fn fat_lto(cgcx: &CodegenContext, } } }).collect::>(); + // Sort the modules to ensure we produce deterministic results. new_modules.sort_by(|module1, module2| module1.1.partial_cmp(&module2.1).unwrap()); serialized_modules.extend(new_modules); serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| { From 36c4873d0934faf56dbfa3e1b84c3a750008c2f8 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Thu, 8 Aug 2019 16:44:42 -0700 Subject: [PATCH 5/6] Try to fix test on Windows. --- src/test/run-make-fulldeps/reproducible-build/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/run-make-fulldeps/reproducible-build/Makefile b/src/test/run-make-fulldeps/reproducible-build/Makefile index addbf9928bf4a..5b9c9d3d03521 100644 --- a/src/test/run-make-fulldeps/reproducible-build/Makefile +++ b/src/test/run-make-fulldeps/reproducible-build/Makefile @@ -81,7 +81,7 @@ extern_flags: fat_lto: rm -rf $(TMPDIR) && mkdir $(TMPDIR) $(RUSTC) reproducible-build-aux.rs - $(RUSTC) reproducible-build.rs -C lto=fat -C opt-level=1 + $(RUSTC) reproducible-build.rs -C lto=fat cp $(TMPDIR)/reproducible-build $(TMPDIR)/reproducible-build-a - $(RUSTC) reproducible-build.rs -C lto=fat -C opt-level=1 + $(RUSTC) reproducible-build.rs -C lto=fat cmp "$(TMPDIR)/reproducible-build-a" "$(TMPDIR)/reproducible-build" || exit 1 From b6767b3096f47b9dc4613cd7ae45c5e89bfd1146 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Fri, 9 Aug 2019 09:24:45 -0700 Subject: [PATCH 6/6] Stop test from running on Windows. --- .../reproducible-build-2/Makefile | 16 +++ .../reproducible-build-2/linker.rs | 44 +++++++ .../reproducible-build-aux.rs | 28 +++++ .../reproducible-build.rs | 116 ++++++++++++++++++ .../reproducible-build/Makefile | 11 +- 5 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 src/test/run-make-fulldeps/reproducible-build-2/Makefile create mode 100644 src/test/run-make-fulldeps/reproducible-build-2/linker.rs create mode 100644 src/test/run-make-fulldeps/reproducible-build-2/reproducible-build-aux.rs create mode 100644 src/test/run-make-fulldeps/reproducible-build-2/reproducible-build.rs diff --git a/src/test/run-make-fulldeps/reproducible-build-2/Makefile b/src/test/run-make-fulldeps/reproducible-build-2/Makefile new file mode 100644 index 0000000000000..b96954fea0d1e --- /dev/null +++ b/src/test/run-make-fulldeps/reproducible-build-2/Makefile @@ -0,0 +1,16 @@ +-include ../tools.mk + +# ignore-musl +# ignore-windows +# Objects are reproducible but their path is not. + +all: \ + fat_lto + +fat_lto: + rm -rf $(TMPDIR) && mkdir $(TMPDIR) + $(RUSTC) reproducible-build-aux.rs + $(RUSTC) reproducible-build.rs -C lto=fat + cp $(TMPDIR)/reproducible-build $(TMPDIR)/reproducible-build-a + $(RUSTC) reproducible-build.rs -C lto=fat + cmp "$(TMPDIR)/reproducible-build-a" "$(TMPDIR)/reproducible-build" || exit 1 diff --git a/src/test/run-make-fulldeps/reproducible-build-2/linker.rs b/src/test/run-make-fulldeps/reproducible-build-2/linker.rs new file mode 100644 index 0000000000000..998d1f328596c --- /dev/null +++ b/src/test/run-make-fulldeps/reproducible-build-2/linker.rs @@ -0,0 +1,44 @@ +use std::env; +use std::path::Path; +use std::fs::File; +use std::io::{Read, Write}; + +fn main() { + let mut dst = env::current_exe().unwrap(); + dst.pop(); + dst.push("linker-arguments1"); + if dst.exists() { + dst.pop(); + dst.push("linker-arguments2"); + assert!(!dst.exists()); + } + + let mut out = String::new(); + for arg in env::args().skip(1) { + let path = Path::new(&arg); + if !path.is_file() { + out.push_str(&arg); + out.push_str("\n"); + continue + } + + let mut contents = Vec::new(); + File::open(path).unwrap().read_to_end(&mut contents).unwrap(); + + out.push_str(&format!("{}: {}\n", arg, hash(&contents))); + } + + File::create(dst).unwrap().write_all(out.as_bytes()).unwrap(); +} + +// fnv hash for now +fn hash(contents: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325; + + for byte in contents { + hash = hash ^ (*byte as u64); + hash = hash.wrapping_mul(0x100000001b3); + } + + hash +} diff --git a/src/test/run-make-fulldeps/reproducible-build-2/reproducible-build-aux.rs b/src/test/run-make-fulldeps/reproducible-build-2/reproducible-build-aux.rs new file mode 100644 index 0000000000000..8105b3d2bda3d --- /dev/null +++ b/src/test/run-make-fulldeps/reproducible-build-2/reproducible-build-aux.rs @@ -0,0 +1,28 @@ +#![crate_type="lib"] + +pub static STATIC: i32 = 1234; + +pub struct Struct { + _t1: std::marker::PhantomData, + _t2: std::marker::PhantomData, +} + +pub fn regular_fn(_: i32) {} + +pub fn generic_fn() {} + +impl Drop for Struct { + fn drop(&mut self) {} +} + +pub enum Enum { + Variant1, + Variant2(u32), + Variant3 { x: u32 } +} + +pub struct TupleStruct(pub i8, pub i16, pub i32, pub i64); + +pub trait Trait { + fn foo(&self); +} diff --git a/src/test/run-make-fulldeps/reproducible-build-2/reproducible-build.rs b/src/test/run-make-fulldeps/reproducible-build-2/reproducible-build.rs new file mode 100644 index 0000000000000..a6c04774c869a --- /dev/null +++ b/src/test/run-make-fulldeps/reproducible-build-2/reproducible-build.rs @@ -0,0 +1,116 @@ +// This test case makes sure that two identical invocations of the compiler +// (i.e., same code base, same compile-flags, same compiler-versions, etc.) +// produce the same output. In the past, symbol names of monomorphized functions +// were not deterministic (which we want to avoid). +// +// The test tries to exercise as many different paths into symbol name +// generation as possible: +// +// - regular functions +// - generic functions +// - methods +// - statics +// - closures +// - enum variant constructors +// - tuple struct constructors +// - drop glue +// - FnOnce adapters +// - Trait object shims +// - Fn Pointer shims + +#![allow(dead_code, warnings)] + +extern crate reproducible_build_aux; + +static STATIC: i32 = 1234; + +pub struct Struct { + x: T1, + y: T2, +} + +fn regular_fn(_: i32) {} + +fn generic_fn() {} + +impl Drop for Struct { + fn drop(&mut self) {} +} + +pub enum Enum { + Variant1, + Variant2(u32), + Variant3 { x: u32 } +} + +struct TupleStruct(i8, i16, i32, i64); + +impl TupleStruct { + pub fn bar(&self) {} +} + +trait Trait { + fn foo(&self); +} + +impl Trait for u64 { + fn foo(&self) {} +} + +impl reproducible_build_aux::Trait for TupleStruct { + fn foo(&self) {} +} + +fn main() { + regular_fn(STATIC); + generic_fn::(); + generic_fn::>(); + generic_fn::, reproducible_build_aux::Struct>(); + + let dropped = Struct { + x: "", + y: 'a', + }; + + let _ = Enum::Variant1; + let _ = Enum::Variant2(0); + let _ = Enum::Variant3 { x: 0 }; + let _ = TupleStruct(1, 2, 3, 4); + + let closure = |x| { + x + 1i32 + }; + + fn inner i32>(f: F) -> i32 { + f(STATIC) + } + + println!("{}", inner(closure)); + + let object_shim: &Trait = &0u64; + object_shim.foo(); + + fn with_fn_once_adapter(f: F) { + f(0); + } + + with_fn_once_adapter(|_:i32| { }); + + reproducible_build_aux::regular_fn(STATIC); + reproducible_build_aux::generic_fn::(); + reproducible_build_aux::generic_fn::>(); + reproducible_build_aux::generic_fn::, + reproducible_build_aux::Struct>(); + + let _ = reproducible_build_aux::Enum::Variant1; + let _ = reproducible_build_aux::Enum::Variant2(0); + let _ = reproducible_build_aux::Enum::Variant3 { x: 0 }; + let _ = reproducible_build_aux::TupleStruct(1, 2, 3, 4); + + let object_shim: &reproducible_build_aux::Trait = &TupleStruct(0, 1, 2, 3); + object_shim.foo(); + + let pointer_shim: &Fn(i32) = ®ular_fn; + + TupleStruct(1, 2, 3, 4).bar(); +} diff --git a/src/test/run-make-fulldeps/reproducible-build/Makefile b/src/test/run-make-fulldeps/reproducible-build/Makefile index 5b9c9d3d03521..a17ec212cfd58 100644 --- a/src/test/run-make-fulldeps/reproducible-build/Makefile +++ b/src/test/run-make-fulldeps/reproducible-build/Makefile @@ -10,8 +10,7 @@ all: \ link_paths \ remap_paths \ different_source_dirs \ - extern_flags \ - fat_lto + extern_flags smoke: rm -rf $(TMPDIR) && mkdir $(TMPDIR) @@ -77,11 +76,3 @@ extern_flags: --extern reproducible_build_aux=$(TMPDIR)/libbar.rlib \ --crate-type rlib cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1 - -fat_lto: - rm -rf $(TMPDIR) && mkdir $(TMPDIR) - $(RUSTC) reproducible-build-aux.rs - $(RUSTC) reproducible-build.rs -C lto=fat - cp $(TMPDIR)/reproducible-build $(TMPDIR)/reproducible-build-a - $(RUSTC) reproducible-build.rs -C lto=fat - cmp "$(TMPDIR)/reproducible-build-a" "$(TMPDIR)/reproducible-build" || exit 1