Skip to content

Commit

Permalink
add allow_fail test attribute
Browse files Browse the repository at this point in the history 
This change allows the user to add an `#[allow_fail]` attribute to
tests that will cause the test to compile & run, but if the test fails
it will not cause the entire test run to fail. The test output will
show the failure, but in yellow instead of red, and also indicate that
it was an allowed failure.
  • Loading branch information
Paul Woolcock committed Jun 24, 2017
1 parent 229d0d3 commit 60dd83e
Show file tree
Hide file tree
Showing 7 changed files with 81 additions and 25 deletions.
41 changes: 23 additions & 18 deletions src/librustdoc/html/markdown.rs
View file
Expand Up @@ -769,7 +769,7 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position
block_info.should_panic, block_info.no_run,
block_info.ignore, block_info.test_harness,
block_info.compile_fail, block_info.error_codes,
line, filename);
line, filename, block_info.allow_fail);
} else {
tests.add_old_test(text, filename);
}
Expand Down Expand Up @@ -859,7 +859,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Sp
block_info.should_panic, block_info.no_run,
block_info.ignore, block_info.test_harness,
block_info.compile_fail, block_info.error_codes,
line, filename);
line, filename, block_info.allow_fail);
prev_offset = offset;
}
Event::Start(Tag::Header(level)) => {
Expand Down Expand Up @@ -889,6 +889,7 @@ struct LangString {
test_harness: bool,
compile_fail: bool,
error_codes: Vec<String>,
allow_fail: bool,
}

impl LangString {
Expand All @@ -902,6 +903,7 @@ impl LangString {
test_harness: false,
compile_fail: false,
error_codes: Vec::new(),
allow_fail: false,
}
}

Expand Down Expand Up @@ -930,6 +932,7 @@ impl LangString {
}
"no_run" => { data.no_run = true; seen_rust_tags = !seen_other_tags; }
"ignore" => { data.ignore = true; seen_rust_tags = !seen_other_tags; }
"allow_fail" => { data.allow_fail = true; seen_rust_tags = !seen_other_tags; }
"rust" => { data.rust = true; seen_rust_tags = true; }
"test_harness" => {
data.test_harness = true;
Expand Down Expand Up @@ -1118,7 +1121,7 @@ mod tests {
fn test_lang_string_parse() {
fn t(s: &str,
should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool,
compile_fail: bool, error_codes: Vec<String>) {
compile_fail: bool, allow_fail: bool, error_codes: Vec<String>) {
assert_eq!(LangString::parse(s), LangString {
should_panic: should_panic,
no_run: no_run,
Expand All @@ -1128,25 +1131,27 @@ mod tests {
compile_fail: compile_fail,
error_codes: error_codes,
original: s.to_owned(),
allow_fail: allow_fail,
})
}

// marker | should_panic| no_run| ignore| rust | test_harness| compile_fail
// | error_codes
t("", false, false, false, true, false, false, Vec::new());
t("rust", false, false, false, true, false, false, Vec::new());
t("sh", false, false, false, false, false, false, Vec::new());
t("ignore", false, false, true, true, false, false, Vec::new());
t("should_panic", true, false, false, true, false, false, Vec::new());
t("no_run", false, true, false, true, false, false, Vec::new());
t("test_harness", false, false, false, true, true, false, Vec::new());
t("compile_fail", false, true, false, true, false, true, Vec::new());
t("{.no_run .example}", false, true, false, true, false, false, Vec::new());
t("{.sh .should_panic}", true, false, false, false, false, false, Vec::new());
t("{.example .rust}", false, false, false, true, false, false, Vec::new());
t("{.test_harness .rust}", false, false, false, true, true, false, Vec::new());
t("text, no_run", false, true, false, false, false, false, Vec::new());
t("text,no_run", false, true, false, false, false, false, Vec::new());
// | allow_fail | error_codes
t("", false, false, false, true, false, false, false, Vec::new());
t("rust", false, false, false, true, false, false, false, Vec::new());
t("sh", false, false, false, false, false, false, false, Vec::new());
t("ignore", false, false, true, true, false, false, false, Vec::new());
t("should_panic", true, false, false, true, false, false, false, Vec::new());
t("no_run", false, true, false, true, false, false, false, Vec::new());
t("test_harness", false, false, false, true, true, false, false, Vec::new());
t("compile_fail", false, true, false, true, false, true, false, Vec::new());
t("allow_fail", false, false, false, true, false, false, true, Vec::new());
t("{.no_run .example}", false, true, false, true, false, false, false, Vec::new());
t("{.sh .should_panic}", true, false, false, false, false, false, false, Vec::new());
t("{.example .rust}", false, false, false, true, false, false, false, Vec::new());
t("{.test_harness .rust}", false, false, false, true, true, false, false, Vec::new());
t("text, no_run", false, true, false, false, false, false, false, Vec::new());
t("text,no_run", false, true, false, false, false, false, false, Vec::new());
}

#[test]
Expand Down
3 changes: 2 additions & 1 deletion src/librustdoc/test.rs
View file
Expand Up @@ -467,7 +467,7 @@ impl Collector {
pub fn add_test(&mut self, test: String,
should_panic: bool, no_run: bool, should_ignore: bool,
as_test_harness: bool, compile_fail: bool, error_codes: Vec<String>,
line: usize, filename: String) {
line: usize, filename: String, allow_fail: bool) {
let name = self.generate_name(line, &filename);
// to be removed when hoedown is removed
if self.render_type == RenderType::Pulldown {
Expand Down Expand Up @@ -499,6 +499,7 @@ impl Collector {
ignore: should_ignore,
// compiler failures are test failures
should_panic: testing::ShouldPanic::No,
allow_fail: allow_fail,
},
testfn: testing::DynTestFn(box move |()| {
let panic = io::set_panic(None);
Expand Down
1 change: 1 addition & 0 deletions src/libsyntax/feature_gate.rs
View file
Expand Up @@ -534,6 +534,7 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
("derive", Normal, Ungated),
("should_panic", Normal, Ungated),
("ignore", Normal, Ungated),
("allow_fail", Normal, Ungated),
("no_implicit_prelude", Normal, Ungated),
("reexport_test_harness_main", Normal, Ungated),
("link_args", Normal, Ungated),
Expand Down
14 changes: 11 additions & 3 deletions src/libsyntax/test.rs
View file
Expand Up @@ -52,7 +52,8 @@ struct Test {
path: Vec<Ident> ,
bench: bool,
ignore: bool,
should_panic: ShouldPanic
should_panic: ShouldPanic,
allow_fail: bool,
}

struct TestCtxt<'a> {
Expand Down Expand Up @@ -133,7 +134,8 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> {
path: self.cx.path.clone(),
bench: is_bench_fn(&self.cx, &i),
ignore: is_ignored(&i),
should_panic: should_panic(&i, &self.cx)
should_panic: should_panic(&i, &self.cx),
allow_fail: is_allowed_fail(&i),
};
self.cx.testfns.push(test);
self.tests.push(i.ident);
Expand Down Expand Up @@ -383,6 +385,10 @@ fn is_ignored(i: &ast::Item) -> bool {
i.attrs.iter().any(|attr| attr.check_name("ignore"))
}

fn is_allowed_fail(i: &ast::Item) -> bool {
i.attrs.iter().any(|attr| attr.check_name("allow_fail"))
}

fn should_panic(i: &ast::Item, cx: &TestCtxt) -> ShouldPanic {
match i.attrs.iter().find(|attr| attr.check_name("should_panic")) {
Some(attr) => {
Expand Down Expand Up @@ -668,14 +674,16 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
}
}
};
let allow_fail_expr = ecx.expr_bool(span, test.allow_fail);

// self::test::TestDesc { ... }
let desc_expr = ecx.expr_struct(
span,
test_path("TestDesc"),
vec![field("name", name_expr),
field("ignore", ignore_expr),
field("should_panic", fail_expr)]);
field("should_panic", fail_expr),
field("allow_fail", allow_fail_expr)]);


let mut visible_path = match cx.toplevel_reexport {
Expand Down
23 changes: 20 additions & 3 deletions src/libtest/lib.rs
View file
Expand Up @@ -212,6 +212,7 @@ pub struct TestDesc {
pub name: TestName,
pub ignore: bool,
pub should_panic: ShouldPanic,
pub allow_fail: bool,
}

#[derive(Clone)]
Expand Down Expand Up @@ -523,6 +524,7 @@ pub enum TestResult {
TrFailed,
TrFailedMsg(String),
TrIgnored,
TrAllowedFail,
TrMetrics(MetricMap),
TrBench(BenchSamples),
}
Expand All @@ -543,6 +545,7 @@ struct ConsoleTestState<T> {
passed: usize,
failed: usize,
ignored: usize,
allowed_fail: usize,
filtered_out: usize,
measured: usize,
metrics: MetricMap,
Expand Down Expand Up @@ -572,6 +575,7 @@ impl<T: Write> ConsoleTestState<T> {
passed: 0,
failed: 0,
ignored: 0,
allowed_fail: 0,
filtered_out: 0,
measured: 0,
metrics: MetricMap::new(),
Expand All @@ -594,6 +598,10 @@ impl<T: Write> ConsoleTestState<T> {
self.write_short_result("ignored", "i", term::color::YELLOW)
}

pub fn write_allowed_fail(&mut self) -> io::Result<()> {
self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW)
}

pub fn write_metric(&mut self) -> io::Result<()> {
self.write_pretty("metric", term::color::CYAN)
}
Expand Down Expand Up @@ -669,6 +677,7 @@ impl<T: Write> ConsoleTestState<T> {
TrOk => self.write_ok(),
TrFailed | TrFailedMsg(_) => self.write_failed(),
TrIgnored => self.write_ignored(),
TrAllowedFail => self.write_allowed_fail(),
TrMetrics(ref mm) => {
self.write_metric()?;
self.write_plain(&format!(": {}\n", mm.fmt_metrics()))
Expand Down Expand Up @@ -702,6 +711,7 @@ impl<T: Write> ConsoleTestState<T> {
TrFailed => "failed".to_owned(),
TrFailedMsg(ref msg) => format!("failed: {}", msg),
TrIgnored => "ignored".to_owned(),
TrAllowedFail => "failed (allowed)".to_owned(),
TrMetrics(ref mm) => mm.fmt_metrics(),
TrBench(ref bs) => fmt_bench_samples(bs),
},
Expand Down Expand Up @@ -761,7 +771,7 @@ impl<T: Write> ConsoleTestState<T> {
}

pub fn write_run_finish(&mut self) -> io::Result<bool> {
assert!(self.passed + self.failed + self.ignored + self.measured == self.total);
assert!(self.passed + self.failed + self.ignored + self.measured + self.allowed_fail == self.total);

if self.options.display_output {
self.write_outputs()?;
Expand All @@ -778,9 +788,10 @@ impl<T: Write> ConsoleTestState<T> {
} else {
self.write_pretty("FAILED", term::color::RED)?;
}
let s = format!(". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
let s = format!(". {} passed; {} failed; {} allowed to fail; {} ignored; {} measured; {} filtered out\n\n",
self.passed,
self.failed,
self.allowed_fail,
self.ignored,
self.measured,
self.filtered_out);
Expand Down Expand Up @@ -891,6 +902,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Resu
st.not_failures.push((test, stdout));
}
TrIgnored => st.ignored += 1,
TrAllowedFail => st.allowed_fail += 1,
TrMetrics(mm) => {
let tname = test.name;
let MetricMap(mm) = mm;
Expand Down Expand Up @@ -1471,8 +1483,13 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> Tes
.unwrap_or(false) {
TrOk
} else {
TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
if desc.allow_fail {
TrAllowedFail
} else {
TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
}
},
_ if desc.allow_fail => TrAllowedFail,
_ => TrFailed,
}
}
Expand Down
23 changes: 23 additions & 0 deletions src/test/run-pass/test-allow-fail-attr.rs
View file
@@ -0,0 +1,23 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// compile-flags: --test

#[test]
#[allow_fail]
fn test1() {
panic!();
}

#[test]
#[allow_fail]
fn test2() {
assert!(true);
}
1 change: 1 addition & 0 deletions src/tools/compiletest/src/main.rs
View file
Expand Up @@ -476,6 +476,7 @@ pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn
name: make_test_name(config, testpaths),
ignore: ignore,
should_panic: should_panic,
allow_fail: false,
},
testfn: make_test_closure(config, testpaths),
}
Expand Down

0 comments on commit 60dd83e

Please sign in to comment.