From f381744d91d214501c96c67b3a5687207ee7ff42 Mon Sep 17 00:00:00 2001 From: Oliver Schneider Date: Mon, 28 Aug 2017 16:54:50 +0200 Subject: [PATCH 1/7] Get the miri test suite to run inside the rustc dev environment --- .gitmodules | 3 ++ config.toml.example | 4 +++ src/bootstrap/bin/rustc.rs | 6 ++++ src/bootstrap/builder.rs | 5 +-- src/bootstrap/check.rs | 44 +++++++++++++++++++++++++ src/bootstrap/config.rs | 4 +++ src/bootstrap/configure.py | 1 + src/bootstrap/mk/Makefile.in | 1 + src/bootstrap/tool.rs | 35 ++++++++++++++++++++ src/ci/docker/x86_64-gnu-aux/Dockerfile | 2 +- src/tools/miri | 1 + src/tools/tidy/src/lib.rs | 1 + 12 files changed, 104 insertions(+), 3 deletions(-) create mode 160000 src/tools/miri diff --git a/.gitmodules b/.gitmodules index d5ae25646547f..2802c8d63913f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -39,3 +39,6 @@ [submodule "src/tools/rustfmt"] path = src/tools/rustfmt url = https://github.com/rust-lang-nursery/rustfmt.git +[submodule "src/tools/miri"] + path = src/tools/miri + url = https://github.com/solson/miri.git diff --git a/config.toml.example b/config.toml.example index 266f425013213..a3790c8d20258 100644 --- a/config.toml.example +++ b/config.toml.example @@ -291,6 +291,10 @@ # When creating source tarballs whether or not to create a source tarball. #dist-src = false +# Whether to also run the Miri tests suite when running tests. +# As a side-effect also generates MIR for all libraries. +#test-miri = false + # ============================================================================= # Options for specific targets # diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index 0baca9e58f4fe..fc27c5d3866c3 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -246,6 +246,12 @@ fn main() { } } + // When running miri tests, we need to generate MIR for all libraries + if env::var("TEST_MIRI").ok().map_or(false, |val| val == "true") && stage != "0" { + cmd.arg("-Zalways-encode-mir"); + cmd.arg("-Zmir-emit-validate=1"); + } + // Force all crates compiled by this compiler to (a) be unstable and (b) // allow the `rustc_private` feature to link to other unstable crates // also in the sysroot. diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 7ff0154bf8bb3..9df7d6c6bef00 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -249,11 +249,11 @@ impl<'a> Builder<'a> { tool::UnstableBookGen, tool::Tidy, tool::Linkchecker, tool::CargoTest, tool::Compiletest, tool::RemoteTestServer, tool::RemoteTestClient, tool::RustInstaller, tool::Cargo, tool::Rls, tool::Rustdoc, tool::Clippy, - native::Llvm, tool::Rustfmt), + native::Llvm, tool::Rustfmt, tool::Miri), Kind::Test => describe!(check::Tidy, check::Bootstrap, check::DefaultCompiletest, check::HostCompiletest, check::Crate, check::CrateLibrustc, check::Rustdoc, check::Linkcheck, check::Cargotest, check::Cargo, check::Rls, check::Docs, - check::ErrorIndex, check::Distcheck, check::Rustfmt), + check::ErrorIndex, check::Distcheck, check::Rustfmt, check::Miri), Kind::Bench => describe!(check::Crate, check::CrateLibrustc), Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook, doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon, @@ -475,6 +475,7 @@ impl<'a> Builder<'a> { } else { PathBuf::from("/path/to/nowhere/rustdoc/not/required") }) + .env("TEST_MIRI", self.config.test_miri.to_string()) .env("RUSTC_FLAGS", self.rustc_flags(target).join(" ")); if mode != Mode::Tool { diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 94bb89145fbf3..d2e0a8e93681c 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -293,6 +293,50 @@ impl Step for Rustfmt { try_run(build, &mut cargo); } } +pub struct Miri { + host: Interned, +} + +impl Step for Miri { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun) -> ShouldRun { + let test_miri = run.builder.build.config.test_miri; + run.path("src/tools/miri").default_condition(test_miri) + } + + fn make_run(run: RunConfig) { + run.builder.ensure(Miri { + host: run.target, + }); + } + + /// Runs `cargo test` for miri. + fn run(self, builder: &Builder) { + let build = builder.build; + let host = self.host; + let compiler = builder.compiler(1, host); + + let miri = builder.ensure(tool::Miri { compiler, target: self.host }); + let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test"); + cargo.arg("--manifest-path").arg(build.src.join("src/tools/miri/Cargo.toml")); + + // Don't build tests dynamically, just a pain to work with + cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1"); + // miri tests need to know about the stage sysroot + cargo.env("MIRI_SYSROOT", builder.sysroot(compiler)); + cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); + cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); + cargo.env("MIRI_PATH", miri); + + builder.add_rustc_lib_path(compiler, &mut cargo); + + try_run(build, &mut cargo); + } +} + fn path_for_cargo(builder: &Builder, compiler: Compiler) -> OsString { // Configure PATH to find the right rustc. NB. we have to use PATH diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 372e0906cc61e..349482eab5f87 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -111,6 +111,7 @@ pub struct Config { pub low_priority: bool, pub channel: String, pub quiet_tests: bool, + pub test_miri: bool, // Fallback musl-root for all targets pub musl_root: Option, pub prefix: Option, @@ -269,6 +270,7 @@ struct Rust { debug: Option, dist_src: Option, quiet_tests: Option, + test_miri: Option, } /// TOML representation of how each build target is configured. @@ -304,6 +306,7 @@ impl Config { config.codegen_tests = true; config.ignore_git = false; config.rust_dist_src = true; + config.test_miri = false; config.on_fail = flags.on_fail; config.stage = flags.stage; @@ -444,6 +447,7 @@ impl Config { set(&mut config.channel, rust.channel.clone()); set(&mut config.rust_dist_src, rust.dist_src); set(&mut config.quiet_tests, rust.quiet_tests); + set(&mut config.test_miri, rust.test_miri); config.rustc_default_linker = rust.default_linker.clone(); config.rustc_default_ar = rust.default_ar.clone(); config.musl_root = rust.musl_root.clone().map(PathBuf::from); diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index fa8b761336068..2438be89776da 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -38,6 +38,7 @@ def v(*args): o("docs", "build.docs", "build standard library documentation") o("compiler-docs", "build.compiler-docs", "build compiler documentation") o("optimize-tests", "rust.optimize-tests", "build tests with optimizations") +o("test-miri", "rust.test-miri", "run miri's test suite") o("debuginfo-tests", "rust.debuginfo-tests", "build tests with debugger metadata") o("quiet-tests", "rust.quiet-tests", "enable quieter output when running tests") o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds") diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 72be9c12e84e6..004f0c31024cd 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -56,6 +56,7 @@ check-aux: src/tools/cargo \ src/tools/rls \ src/tools/rustfmt \ + src/tools/miri \ src/test/pretty \ src/test/run-pass/pretty \ src/test/run-fail/pretty \ diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index d082012acc191..541247804857b 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -479,6 +479,41 @@ impl Step for Rustfmt { } } + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Miri { + pub compiler: Compiler, + pub target: Interned, +} + +impl Step for Miri { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun) -> ShouldRun { + let builder = run.builder; + run.path("src/tools/miri").default_condition(builder.build.config.test_miri) + } + + fn make_run(run: RunConfig) { + run.builder.ensure(Miri { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.build.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder) -> PathBuf { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "miri", + mode: Mode::Librustc, + path: "src/tools/miri", + }) + } +} + impl<'a> Builder<'a> { /// Get a `Command` which is ready to run `tool` in `stage` built for /// `host`. diff --git a/src/ci/docker/x86_64-gnu-aux/Dockerfile b/src/ci/docker/x86_64-gnu-aux/Dockerfile index 35a387221c644..a453c62cc9e83 100644 --- a/src/ci/docker/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/x86_64-gnu-aux/Dockerfile @@ -17,5 +17,5 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu +ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-test-miri ENV RUST_CHECK_TARGET check-aux diff --git a/src/tools/miri b/src/tools/miri new file mode 160000 index 0000000000000..ce3576f7d8193 --- /dev/null +++ b/src/tools/miri @@ -0,0 +1 @@ +Subproject commit ce3576f7d81931f77264f85a3c68077605310019 diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs index 56c0b21cd53c7..90bf7a5e0a687 100644 --- a/src/tools/tidy/src/lib.rs +++ b/src/tools/tidy/src/lib.rs @@ -65,6 +65,7 @@ fn filter_dirs(path: &Path) -> bool { "src/tools/clippy", "src/tools/rust-installer", "src/tools/rustfmt", + "src/tools/miri", ]; skip.iter().any(|p| path.ends_with(p)) } From f0b5402283f131f04132d77babecc41949fe35c8 Mon Sep 17 00:00:00 2001 From: Oliver Schneider Date: Mon, 11 Sep 2017 11:00:51 +0200 Subject: [PATCH 2/7] Improve documentation --- src/ci/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md index 627b5062df333..adce6a00d4623 100644 --- a/src/ci/docker/README.md +++ b/src/ci/docker/README.md @@ -20,7 +20,7 @@ Images will output artifacts in an `obj` dir at the root of a repository. - Each directory, excluding `scripts` and `disabled`, corresponds to a docker image - `scripts` contains files shared by docker images -- `disabled` contains images that are not build travis +- `disabled` contains images that are not built on travis ## Cross toolchains From ab018c76e14b87f3c9e0b7384cc9b02d94779cd5 Mon Sep 17 00:00:00 2001 From: Oliver Schneider Date: Wed, 30 Aug 2017 18:59:26 +0200 Subject: [PATCH 3/7] Add a file to trivially disable tool building or testing --- src/bootstrap/check.rs | 19 +++++--- src/bootstrap/config.rs | 15 +++++++ src/bootstrap/lib.rs | 29 +++++++----- src/bootstrap/tool.rs | 16 +++++-- src/bootstrap/toolstate.rs | 48 ++++++++++++++++++++ src/build_helper/lib.rs | 80 ++++++++++++++++++++++++++-------- src/liballoc_jemalloc/build.rs | 6 +-- src/libstd/build.rs | 9 ++-- src/tools/toolstate.toml | 24 ++++++++++ 9 files changed, 203 insertions(+), 43 deletions(-) create mode 100644 src/bootstrap/toolstate.rs create mode 100644 src/tools/toolstate.toml diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index d2e0a8e93681c..78ec2579c4aa7 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -23,7 +23,7 @@ use std::path::{PathBuf, Path}; use std::process::Command; use std::io::Read; -use build_helper::{self, output}; +use build_helper::{self, output, BuildExpectation}; use builder::{Kind, RunConfig, ShouldRun, Builder, Compiler, Step}; use cache::{INTERNER, Interned}; @@ -33,6 +33,7 @@ use native; use tool::{self, Tool}; use util::{self, dylib_path, dylib_path_var}; use {Build, Mode}; +use toolstate::ToolState; const ADB_TEST_DIR: &str = "/data/tmp/work"; @@ -64,17 +65,21 @@ impl fmt::Display for TestKind { } } -fn try_run(build: &Build, cmd: &mut Command) { +fn try_run_expecting(build: &Build, cmd: &mut Command, expect: BuildExpectation) { if !build.fail_fast { - if !build.try_run(cmd) { + if !build.try_run(cmd, expect) { let failures = build.delayed_failures.get(); build.delayed_failures.set(failures + 1); } } else { - build.run(cmd); + build.run_expecting(cmd, expect); } } +fn try_run(build: &Build, cmd: &mut Command) { + try_run_expecting(build, cmd, BuildExpectation::None) +} + fn try_run_quiet(build: &Build, cmd: &mut Command) { if !build.fail_fast { if !build.try_run_quiet(cmd) { @@ -333,7 +338,11 @@ impl Step for Miri { builder.add_rustc_lib_path(compiler, &mut cargo); - try_run(build, &mut cargo); + try_run_expecting( + build, + &mut cargo, + builder.build.config.toolstate.miri.passes(ToolState::Testing), + ); } } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 349482eab5f87..c8b2ed042c119 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -27,6 +27,7 @@ use util::exe; use cache::{INTERNER, Interned}; use flags::Flags; pub use flags::Subcommand; +use toolstate::ToolStates; /// Global configuration for the entire build and/or bootstrap. /// @@ -131,6 +132,8 @@ pub struct Config { // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, + + pub toolstate: ToolStates, } /// Per-target configuration stored in the global configuration structure. @@ -333,6 +336,18 @@ impl Config { } }).unwrap_or_else(|| TomlConfig::default()); + let toolstate_toml_path = config.src.join("src/tools/toolstate.toml"); + let parse_toolstate = || -> Result<_, Box<::std::error::Error>> { + let mut f = File::open(toolstate_toml_path)?; + let mut contents = String::new(); + f.read_to_string(&mut contents)?; + Ok(toml::from_str(&contents)?) + }; + config.toolstate = parse_toolstate().unwrap_or_else(|err| { + println!("failed to parse TOML configuration 'toolstate.toml': {}", err); + process::exit(2); + }); + let build = toml.build.clone().unwrap_or(Build::default()); set(&mut config.build, build.build.clone().map(|x| INTERNER.intern_string(x))); set(&mut config.build, flags.build); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 67791e8758c0b..6bca17c8ba860 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -143,7 +143,8 @@ use std::path::{PathBuf, Path}; use std::process::Command; use std::slice; -use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime}; +use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime, + BuildExpectation}; use util::{exe, libdir, OutputFolder, CiEnv}; @@ -164,6 +165,7 @@ pub mod util; mod builder; mod cache; mod tool; +mod toolstate; #[cfg(windows)] mod job; @@ -542,24 +544,31 @@ impl Build { .join(libdir(&self.config.build)) } + /// Runs a command, printing out nice contextual information if its build + /// status is not the expected one + fn run_expecting(&self, cmd: &mut Command, expect: BuildExpectation) { + self.verbose(&format!("running: {:?}", cmd)); + run_silent(cmd, expect) + } + /// Runs a command, printing out nice contextual information if it fails. fn run(&self, cmd: &mut Command) { - self.verbose(&format!("running: {:?}", cmd)); - run_silent(cmd) + self.run_expecting(cmd, BuildExpectation::None) } /// Runs a command, printing out nice contextual information if it fails. fn run_quiet(&self, cmd: &mut Command) { self.verbose(&format!("running: {:?}", cmd)); - run_suppressed(cmd) + run_suppressed(cmd, BuildExpectation::None) } - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - fn try_run(&self, cmd: &mut Command) -> bool { + /// Runs a command, printing out nice contextual information if its build + /// status is not the expected one. + /// Exits if the command failed to execute at all, otherwise returns whether + /// the expectation was met + fn try_run(&self, cmd: &mut Command, expect: BuildExpectation) -> bool { self.verbose(&format!("running: {:?}", cmd)); - try_run_silent(cmd) + try_run_silent(cmd, expect) } /// Runs a command, printing out nice contextual information if it fails. @@ -567,7 +576,7 @@ impl Build { /// `status.success()`. fn try_run_quiet(&self, cmd: &mut Command) -> bool { self.verbose(&format!("running: {:?}", cmd)); - try_run_suppressed(cmd) + try_run_suppressed(cmd, BuildExpectation::None) } pub fn is_verbose(&self) -> bool { diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 541247804857b..4e422473b9ad4 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -21,6 +21,8 @@ use compile::{self, libtest_stamp, libstd_stamp, librustc_stamp}; use native; use channel::GitInfo; use cache::Interned; +use toolstate::ToolState; +use build_helper::BuildExpectation; #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct CleanTools { @@ -64,6 +66,7 @@ struct ToolBuild { tool: &'static str, path: &'static str, mode: Mode, + expectation: BuildExpectation, } impl Step for ToolBuild { @@ -83,6 +86,7 @@ impl Step for ToolBuild { let target = self.target; let tool = self.tool; let path = self.path; + let expectation = self.expectation; match self.mode { Mode::Libstd => builder.ensure(compile::Std { compiler, target }), @@ -95,7 +99,7 @@ impl Step for ToolBuild { println!("Building stage{} tool {} ({})", compiler.stage, tool, target); let mut cargo = prepare_tool_cargo(builder, compiler, target, "build", path); - build.run(&mut cargo); + build.run_expecting(&mut cargo, expectation); build.cargo_out(compiler, Mode::Tool, target).join(exe(tool, &compiler.host)) } } @@ -200,6 +204,7 @@ macro_rules! tool { tool: $tool_name, mode: $mode, path: $path, + expectation: BuildExpectation::None, }) } } @@ -247,6 +252,7 @@ impl Step for RemoteTestServer { tool: "remote-test-server", mode: Mode::Libstd, path: "src/tools/remote-test-server", + expectation: BuildExpectation::None, }) } } @@ -359,6 +365,7 @@ impl Step for Cargo { tool: "cargo", mode: Mode::Librustc, path: "src/tools/cargo", + expectation: BuildExpectation::None, }) } } @@ -398,6 +405,7 @@ impl Step for Clippy { tool: "clippy", mode: Mode::Librustc, path: "src/tools/clippy", + expectation: BuildExpectation::None, }) } } @@ -441,6 +449,7 @@ impl Step for Rls { tool: "rls", mode: Mode::Librustc, path: "src/tools/rls", + expectation: BuildExpectation::None, }) } } @@ -492,8 +501,8 @@ impl Step for Miri { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - let builder = run.builder; - run.path("src/tools/miri").default_condition(builder.build.config.test_miri) + let build_miri = run.builder.build.config.test_miri; + run.path("src/tools/miri").default_condition(build_miri) } fn make_run(run: RunConfig) { @@ -510,6 +519,7 @@ impl Step for Miri { tool: "miri", mode: Mode::Librustc, path: "src/tools/miri", + expectation: builder.build.config.toolstate.miri.passes(ToolState::Compiling), }) } } diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs new file mode 100644 index 0000000000000..9556a8b52df67 --- /dev/null +++ b/src/bootstrap/toolstate.rs @@ -0,0 +1,48 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use build_helper::BuildExpectation; + +#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq)] +/// Whether a tool can be compiled, tested or neither +pub enum ToolState { + /// The tool compiles successfully, but the test suite fails + Compiling = 1, + /// The tool compiles successfully and its test suite passes + Testing = 2, + /// The tool can't even be compiled + Broken = 0, +} + +impl ToolState { + /// If a tool with the current toolstate should be working on + /// the given toolstate + pub fn passes(self, other: ToolState) -> BuildExpectation { + if self as usize >= other as usize { + BuildExpectation::Succeeding + } else { + BuildExpectation::Failing + } + } +} + +impl Default for ToolState { + fn default() -> Self { + // err on the safe side + ToolState::Broken + } +} + +#[derive(Copy, Clone, Debug, Deserialize, Default)] +/// Used to express which tools should (not) be compiled or tested. +/// This is created from `toolstate.toml`. +pub struct ToolStates { + pub miri: ToolState, +} diff --git a/src/build_helper/lib.rs b/src/build_helper/lib.rs index 8b4c7f2ac3177..e81dab70b43e7 100644 --- a/src/build_helper/lib.rs +++ b/src/build_helper/lib.rs @@ -35,55 +35,97 @@ macro_rules! t { }) } -pub fn run(cmd: &mut Command) { +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum BuildExpectation { + Succeeding, + Failing, + None, +} + +pub fn run(cmd: &mut Command, expect: BuildExpectation) { println!("running: {:?}", cmd); - run_silent(cmd); + run_silent(cmd, expect); } -pub fn run_silent(cmd: &mut Command) { - if !try_run_silent(cmd) { +pub fn run_silent(cmd: &mut Command, expect: BuildExpectation) { + if !try_run_silent(cmd, expect) { std::process::exit(1); } } -pub fn try_run_silent(cmd: &mut Command) -> bool { +pub fn try_run_silent(cmd: &mut Command, expect: BuildExpectation) -> bool { let status = match cmd.status() { Ok(status) => status, Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), }; - if !status.success() { - println!("\n\ncommand did not execute successfully: {:?}\n\ - expected success, got: {}\n\n", - cmd, - status); + process_status( + cmd, + status.success(), + expect, + || println!("\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n", + cmd, + status)) +} + +fn process_status( + cmd: &Command, + success: bool, + expect: BuildExpectation, + f: F, +) -> bool { + use BuildExpectation::*; + match (expect, success) { + (None, false) => { f(); false }, + // Non-tool build succeeds, everything is good + (None, true) => true, + // Tool expected to work and is working + (Succeeding, true) => true, + // Tool expected to fail and is failing + (Failing, false) => { + println!("This failure is expected (see `src/tools/toolstate.toml`)"); + true + }, + // Tool expected to work, but is failing + (Succeeding, false) => { + f(); + println!("You can disable the tool in `src/tools/toolstate.toml`"); + false + }, + // Tool expected to fail, but is working + (Failing, true) => { + println!("Expected `{:?}` to fail, but it succeeded.\n\ + Please adjust `src/tools/toolstate.toml` accordingly", cmd); + false + } } - status.success() } -pub fn run_suppressed(cmd: &mut Command) { - if !try_run_suppressed(cmd) { +pub fn run_suppressed(cmd: &mut Command, expect: BuildExpectation) { + if !try_run_suppressed(cmd, expect) { std::process::exit(1); } } -pub fn try_run_suppressed(cmd: &mut Command) -> bool { +pub fn try_run_suppressed(cmd: &mut Command, expect: BuildExpectation) -> bool { let output = match cmd.output() { Ok(status) => status, Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), }; - if !output.status.success() { - println!("\n\ncommand did not execute successfully: {:?}\n\ + process_status( + cmd, + output.status.success(), + expect, + || println!("\n\ncommand did not execute successfully: {:?}\n\ expected success, got: {}\n\n\ stdout ----\n{}\n\ stderr ----\n{}\n\n", cmd, output.status, String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr)); - } - output.status.success() + String::from_utf8_lossy(&output.stderr))) } pub fn gnu_target(target: &str) -> String { diff --git a/src/liballoc_jemalloc/build.rs b/src/liballoc_jemalloc/build.rs index 1864df4477abf..d89d3bcdb62a5 100644 --- a/src/liballoc_jemalloc/build.rs +++ b/src/liballoc_jemalloc/build.rs @@ -16,7 +16,7 @@ extern crate gcc; use std::env; use std::path::PathBuf; use std::process::Command; -use build_helper::{run, native_lib_boilerplate}; +use build_helper::{run, native_lib_boilerplate, BuildExpectation}; fn main() { // FIXME: This is a hack to support building targets that don't @@ -126,7 +126,7 @@ fn main() { cmd.arg("--with-lg-quantum=4"); } - run(&mut cmd); + run(&mut cmd, BuildExpectation::None); let mut make = Command::new(build_helper::make(&host)); make.current_dir(&native.out_dir) @@ -143,7 +143,7 @@ fn main() { .arg(env::var("NUM_JOBS").expect("NUM_JOBS was not set")); } - run(&mut make); + run(&mut make, BuildExpectation::None); // The pthread_atfork symbols is used by jemalloc on android but the really // old android we're building on doesn't have them defined, so just make diff --git a/src/libstd/build.rs b/src/libstd/build.rs index ebf07eb0423a0..19ea25fc7df07 100644 --- a/src/libstd/build.rs +++ b/src/libstd/build.rs @@ -15,7 +15,7 @@ extern crate gcc; use std::env; use std::process::Command; -use build_helper::{run, native_lib_boilerplate}; +use build_helper::{run, native_lib_boilerplate, BuildExpectation}; fn main() { let target = env::var("TARGET").expect("TARGET was not set"); @@ -97,11 +97,14 @@ fn build_libbacktrace(host: &str, target: &str) -> Result<(), ()> { .env("CC", compiler.path()) .env("AR", &ar) .env("RANLIB", format!("{} s", ar.display())) - .env("CFLAGS", cflags)); + .env("CFLAGS", cflags), + BuildExpectation::None); run(Command::new(build_helper::make(host)) .current_dir(&native.out_dir) .arg(format!("INCDIR={}", native.src_dir.display())) - .arg("-j").arg(env::var("NUM_JOBS").expect("NUM_JOBS was not set"))); + .arg("-j").arg(env::var("NUM_JOBS").expect("NUM_JOBS was not set")), + BuildExpectation::None); + Ok(()) } diff --git a/src/tools/toolstate.toml b/src/tools/toolstate.toml new file mode 100644 index 0000000000000..697be4efadbc8 --- /dev/null +++ b/src/tools/toolstate.toml @@ -0,0 +1,24 @@ +# This file reflects the current status of all tools which are allowed +# to fail without failing the build. +# +# There are three states a tool can be in: +# 1. Broken: The tool doesn't build +# 2. Building: The tool builds but its tests are failing +# 3. Testing: The tool builds and its tests are passing +# +# In the future there will be further states like "Distributing", which +# configures whether the tool is included in the Rust distribution. +# +# If a tool was working before your PR but is broken now, consider +# updating the tool within your PR. How to do that is described in +# "CONTRIBUTING.md#External Dependencies". If the effort required is not +# warranted (e.g. due to the tool abusing some API that you changed, and +# fixing the tool would mean a significant refactoring), you can disable +# the tool here, by changing its state to `Broken`. Remember to ping +# the tool authors if you do not fix their tool, so they can proactively +# fix it, instead of being surprised by the breakage. +# +# Each tool has a list of people to ping + +# ping @oli-obk @RalfJung @eddyb +miri = "Testing" From 13921dafbff6dcff629164fda7ab454615301877 Mon Sep 17 00:00:00 2001 From: Oliver Schneider Date: Tue, 5 Sep 2017 17:22:52 +0200 Subject: [PATCH 4/7] -Zmir-emit-validate is in stage 0 --- src/bootstrap/bin/rustc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index fc27c5d3866c3..848b10d312cea 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -247,7 +247,7 @@ fn main() { } // When running miri tests, we need to generate MIR for all libraries - if env::var("TEST_MIRI").ok().map_or(false, |val| val == "true") && stage != "0" { + if env::var("TEST_MIRI").ok().map_or(false, |val| val == "true") { cmd.arg("-Zalways-encode-mir"); cmd.arg("-Zmir-emit-validate=1"); } From 68fc65eaf2a17c5fad14911b15f8a13d5c3d5b4a Mon Sep 17 00:00:00 2001 From: Oliver Schneider Date: Sun, 10 Sep 2017 11:48:56 +0200 Subject: [PATCH 5/7] Prevent distribution if miri is enabled --- src/bootstrap/builder.rs | 2 +- src/bootstrap/dist.rs | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 9df7d6c6bef00..1af96f1bf0b8f 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -260,7 +260,7 @@ impl<'a> Builder<'a> { doc::Reference, doc::Rustdoc, doc::CargoBook), Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts, dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo, - dist::Rls, dist::Extended, dist::HashSign), + dist::Rls, dist::Extended, dist::HashSign, dist::DontDistWithMiriEnabled), Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls, install::Analysis, install::Src, install::Rustc), } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 7bca088dbd5a8..5188604b0a69a 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1111,6 +1111,31 @@ impl Step for Rls { } } + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct DontDistWithMiriEnabled; + +impl Step for DontDistWithMiriEnabled { + type Output = PathBuf; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun) -> ShouldRun { + let build_miri = run.builder.build.config.test_miri; + run.default_condition(build_miri) + } + + fn make_run(run: RunConfig) { + run.builder.ensure(DontDistWithMiriEnabled); + } + + fn run(self, _: &Builder) -> PathBuf { + panic!("Do not distribute with miri enabled.\n\ + The distributed libraries would include all MIR (increasing binary size). + The distributed MIR would include validation statements."); + } +} + + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Extended { stage: u32, From a9df19b42f86a2c98e4de1c2ae15e02ffe338792 Mon Sep 17 00:00:00 2001 From: Oliver Schneider Date: Sun, 17 Sep 2017 21:42:35 +0200 Subject: [PATCH 6/7] Update miri submodule --- src/tools/miri | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/miri b/src/tools/miri index ce3576f7d8193..80853e2f24a01 160000 --- a/src/tools/miri +++ b/src/tools/miri @@ -1 +1 @@ -Subproject commit ce3576f7d81931f77264f85a3c68077605310019 +Subproject commit 80853e2f24a01db96fe9821e468dd2af75a4d2e5 From 01555b1da1bb0fb289b1c971a046b315c751c861 Mon Sep 17 00:00:00 2001 From: Oliver Schneider Date: Sun, 17 Sep 2017 21:45:54 +0200 Subject: [PATCH 7/7] Rebase fallout --- src/bootstrap/check.rs | 2 ++ src/bootstrap/tool.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 78ec2579c4aa7..95ac2be5423c6 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -298,6 +298,8 @@ impl Step for Rustfmt { try_run(build, &mut cargo); } } + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct Miri { host: Interned, } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 4e422473b9ad4..1617351af4cf1 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -484,6 +484,7 @@ impl Step for Rustfmt { tool: "rustfmt", mode: Mode::Librustc, path: "src/tools/rustfmt", + expectation: BuildExpectation::None, }) } }