Skip to content

Commit

Permalink
Auto merge of #32016 - nikomatsakis:incr-comp-save, r=mw
Browse files Browse the repository at this point in the history 
Save/load incremental compilation dep graph

Contains the code to serialize/deserialize the dep graph to disk between executions. We also hash the item contents and compare to the new hashes. Also includes a unit test harness. There are definitely some known limitations, such as #32014 and #32015, but I am leaving those for follow-up work.

Note that this PR builds on #32007, so the overlapping commits can be excluded from review.

r? @michaelwoerister
  • Loading branch information
@bors
bors committed Apr 7, 2016
2 parents 470ca1c + 9eaae92 commit 7979dd6
Show file tree
Hide file tree
Showing 56 changed files with 2,053 additions and 712 deletions.
8 changes: 5 additions & 3 deletions mk/crates.mk
View file
Expand Up @@ -58,7 +58,7 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
rustc_data_structures rustc_platform_intrinsics \
rustc_plugin rustc_metadata rustc_passes rustc_save_analysis \
rustc_const_eval rustc_const_math
rustc_const_eval rustc_const_math rustc_incremental
HOST_CRATES := syntax syntax_ext $(RUSTC_CRATES) rustdoc fmt_macros \
flate arena graphviz rbml log serialize
TOOLS := compiletest rustdoc rustc rustbook error_index_generator
Expand Down Expand Up @@ -105,7 +105,8 @@ DEPS_rustc_data_structures := std log serialize
DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_borrowck \
rustc_typeck rustc_mir rustc_resolve log syntax serialize rustc_llvm \
rustc_trans rustc_privacy rustc_lint rustc_plugin \
rustc_metadata syntax_ext rustc_passes rustc_save_analysis rustc_const_eval
rustc_metadata syntax_ext rustc_passes rustc_save_analysis rustc_const_eval \
rustc_incremental
DEPS_rustc_lint := rustc log syntax rustc_const_eval
DEPS_rustc_llvm := native:rustllvm libc std rustc_bitflags
DEPS_rustc_metadata := rustc syntax rbml rustc_const_math
Expand All @@ -117,7 +118,8 @@ DEPS_rustc_plugin := rustc rustc_metadata syntax rustc_mir
DEPS_rustc_privacy := rustc log syntax
DEPS_rustc_trans := arena flate getopts graphviz libc rustc rustc_back rustc_mir \
log syntax serialize rustc_llvm rustc_platform_intrinsics \
rustc_const_math rustc_const_eval
rustc_const_math rustc_const_eval rustc_incremental
DEPS_rustc_incremental := rbml rustc serialize rustc_data_structures
DEPS_rustc_save_analysis := rustc log syntax
DEPS_rustc_typeck := rustc syntax rustc_platform_intrinsics rustc_const_math \
rustc_const_eval
Expand Down
12 changes: 11 additions & 1 deletion mk/tests.mk
View file
Expand Up @@ -305,6 +305,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
check-stage$(1)-T-$(2)-H-$(3)-doc-crates-exec \
check-stage$(1)-T-$(2)-H-$(3)-debuginfo-gdb-exec \
check-stage$(1)-T-$(2)-H-$(3)-debuginfo-lldb-exec \
check-stage$(1)-T-$(2)-H-$(3)-incremental-exec \
check-stage$(1)-T-$(2)-H-$(3)-doc-exec \
check-stage$(1)-T-$(2)-H-$(3)-pretty-exec

Expand Down Expand Up @@ -481,6 +482,7 @@ DEBUGINFO_LLDB_RS := $(call rwildcard,$(S)src/test/debuginfo/,*.rs)
CODEGEN_RS := $(call rwildcard,$(S)src/test/codegen/,*.rs)
CODEGEN_CC := $(call rwildcard,$(S)src/test/codegen/,*.cc)
CODEGEN_UNITS_RS := $(call rwildcard,$(S)src/test/codegen-units/,*.rs)
INCREMENTAL_RS := $(call rwildcard,$(S)src/test/incremental/,*.rs)
RUSTDOCCK_RS := $(call rwildcard,$(S)src/test/rustdoc/,*.rs)

RPASS_TESTS := $(RPASS_RS)
Expand All @@ -496,6 +498,7 @@ DEBUGINFO_GDB_TESTS := $(DEBUGINFO_GDB_RS)
DEBUGINFO_LLDB_TESTS := $(DEBUGINFO_LLDB_RS)
CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
CODEGEN_UNITS_TESTS := $(CODEGEN_UNITS_RS)
INCREMENTAL_TESTS := $(INCREMENTAL_RS)
RUSTDOCCK_TESTS := $(RUSTDOCCK_RS)

CTEST_SRC_BASE_rpass = run-pass
Expand Down Expand Up @@ -558,6 +561,11 @@ CTEST_BUILD_BASE_codegen-units = codegen-units
CTEST_MODE_codegen-units = codegen-units
CTEST_RUNTOOL_codegen-units = $(CTEST_RUNTOOL)

CTEST_SRC_BASE_incremental = incremental
CTEST_BUILD_BASE_incremental = incremental
CTEST_MODE_incremental = incremental
CTEST_RUNTOOL_incremental = $(CTEST_RUNTOOL)

CTEST_SRC_BASE_rustdocck = rustdoc
CTEST_BUILD_BASE_rustdocck = rustdoc
CTEST_MODE_rustdocck = rustdoc
Expand Down Expand Up @@ -681,6 +689,7 @@ CTEST_DEPS_debuginfo-lldb_$(1)-T-$(2)-H-$(3) = $$(DEBUGINFO_LLDB_TESTS) \
$(S)src/etc/lldb_rust_formatters.py
CTEST_DEPS_codegen_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_TESTS)
CTEST_DEPS_codegen-units_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_UNITS_TESTS)
CTEST_DEPS_incremental_$(1)-T-$(2)-H-$(3) = $$(INCREMENTAL_TESTS)
CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
$(S)src/etc/htmldocck.py
Expand Down Expand Up @@ -747,7 +756,7 @@ endif
endef

CTEST_NAMES = rpass rpass-valgrind rpass-full rfail-full cfail-full rfail cfail pfail \
debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck
debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck incremental

$(foreach host,$(CFG_HOST), \
$(eval $(foreach target,$(CFG_TARGET), \
Expand Down Expand Up @@ -945,6 +954,7 @@ TEST_GROUPS = \
debuginfo-lldb \
codegen \
codegen-units \
incremental \
doc \
$(foreach docname,$(DOC_NAMES),doc-$(docname)) \
pretty \
Expand Down
5 changes: 4 additions & 1 deletion src/compiletest/common.rs
View file
Expand Up @@ -25,7 +25,8 @@ pub enum Mode {
DebugInfoLldb,
Codegen,
Rustdoc,
CodegenUnits
CodegenUnits,
Incremental,
}

impl FromStr for Mode {
Expand All @@ -43,6 +44,7 @@ impl FromStr for Mode {
"codegen" => Ok(Codegen),
"rustdoc" => Ok(Rustdoc),
"codegen-units" => Ok(CodegenUnits),
"incremental" => Ok(Incremental),
_ => Err(()),
}
}
Expand All @@ -62,6 +64,7 @@ impl fmt::Display for Mode {
Codegen => "codegen",
Rustdoc => "rustdoc",
CodegenUnits => "codegen-units",
Incremental => "incremental",
}, f)
}
}
Expand Down
3 changes: 2 additions & 1 deletion src/compiletest/compiletest.rs
View file
Expand Up @@ -71,7 +71,8 @@ pub fn parse_config(args: Vec<String> ) -> Config {
reqopt("", "aux-base", "directory to find auxiliary test files", "PATH"),
reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
reqopt("", "mode", "which sort of compile tests to run",
"(compile-fail|parse-fail|run-fail|run-pass|run-pass-valgrind|pretty|debug-info)"),
"(compile-fail|parse-fail|run-fail|run-pass|\
run-pass-valgrind|pretty|debug-info|incremental)"),
optflag("", "ignored", "run tests marked as ignored"),
optopt("", "runtool", "supervisor program to run tests under \
(eg. emulator, valgrind)", "PROGRAM"),
Expand Down
66 changes: 66 additions & 0 deletions src/compiletest/runtest.rs
View file
Expand Up @@ -11,6 +11,7 @@
use common::Config;
use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
use common::{Incremental};
use errors::{self, ErrorKind};
use header::TestProps;
use header;
Expand Down Expand Up @@ -59,6 +60,7 @@ pub fn run(config: Config, testpaths: &TestPaths) {
Codegen => run_codegen_test(&config, &props, &testpaths),
Rustdoc => run_rustdoc_test(&config, &props, &testpaths),
CodegenUnits => run_codegen_units_test(&config, &props, &testpaths),
Incremental => run_incremental_test(&config, &props, &testpaths),
}
}

Expand Down Expand Up @@ -1966,3 +1968,67 @@ fn run_codegen_units_test(config: &Config, props: &TestProps, testpaths: &TestPa
panic!();
}
}

fn run_incremental_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
// Basic plan for a test incremental/foo/bar.rs:
// - load list of revisions pass1, fail2, pass3
// - each should begin with `rpass`, `rfail`, or `cfail`
// - if `rpass`, expect compile and execution to succeed
// - if `cfail`, expect compilation to fail
// - if `rfail`, expect execution to fail
// - create a directory build/foo/bar.incremental
// - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C pass1
// - because name of revision starts with "pass", expect success
// - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C fail2
// - because name of revision starts with "fail", expect an error
// - load expected errors as usual, but filter for those that end in `[fail2]`
// - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C pass3
// - because name of revision starts with "pass", expect success
// - execute build/foo/bar.exe and save output
//
// FIXME -- use non-incremental mode as an oracle? That doesn't apply
// to #[rustc_dirty] and clean tests I guess

assert!(!props.revisions.is_empty(), "incremental tests require a list of revisions");

let output_base_name = output_base_name(config, testpaths);

// Create the incremental workproduct directory.
let incremental_dir = output_base_name.with_extension("incremental");
if incremental_dir.exists() {
fs::remove_dir_all(&incremental_dir).unwrap();
}
fs::create_dir_all(&incremental_dir).unwrap();

if config.verbose {
print!("incremental_dir={}", incremental_dir.display());
}

for revision in &props.revisions {
let mut revision_props = props.clone();
header::load_props_into(&mut revision_props, &testpaths.file, Some(&revision));

revision_props.compile_flags.extend(vec![
format!("-Z"),
format!("incremental={}", incremental_dir.display()),
format!("--cfg"),
format!("{}", revision),
]);

if config.verbose {
print!("revision={:?} revision_props={:#?}", revision, revision_props);
}

if revision.starts_with("rpass") {
run_rpass_test_revision(config, &revision_props, testpaths, Some(&revision));
} else if revision.starts_with("rfail") {
run_rfail_test_revision(config, &revision_props, testpaths, Some(&revision));
} else if revision.starts_with("cfail") {
run_cfail_test_revision(config, &revision_props, testpaths, Some(&revision));
} else {
fatal(
Some(revision),
"revision name must begin with rpass, rfail, or cfail");
}
}
}

0 comments on commit 7979dd6

Please sign in to comment.