Skip to content
Permalink
Browse files

do not print panic message on doctest failures

  • Loading branch information...
euclio committed May 4, 2019
1 parent 8dd4aae commit f1c60e705465a691bf6660205ea073acefc92c51
@@ -20,7 +20,7 @@ use std::io::prelude::*;
use std::io;
use std::panic::{self, AssertUnwindSafe};
use std::path::PathBuf;
use std::process::Command;
use std::process::{self, Command};
use std::str;
use std::sync::{Arc, Mutex};

@@ -159,13 +159,45 @@ fn scrape_test_config(krate: &::rustc::hir::Crate) -> TestOptions {
opts
}

fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
cfgs: Vec<String>, libs: Vec<SearchPath>,
cg: CodegenOptions, externs: Externs,
should_panic: bool, no_run: bool, as_test_harness: bool,
compile_fail: bool, mut error_codes: Vec<String>, opts: &TestOptions,
maybe_sysroot: Option<PathBuf>, linker: Option<PathBuf>, edition: Edition,
persist_doctests: Option<PathBuf>) {
/// Documentation test failure modes.
enum TestFailure {
/// The test failed to compile.
CompileError,
/// The test is marked `compile_fail` but compiled successfully.
UnexpectedCompilePass,
/// The test failed to compile (as expected) but the compiler output did not contain all
/// expected error codes.
MissingErrorCodes(Vec<String>),
/// The test binary was unable to be executed.
ExecutionError(io::Error),
/// The test binary exited with a non-zero exit code.
///
/// This typically means an assertion in the test failed or another form of panic occurred.
ExecutionFailure(process::Output),
/// The test is marked `should_panic` but the test binary executed successfully.
UnexpectedRunPass,
}

fn run_test(
test: &str,
cratename: &str,
filename: &FileName,
line: usize,
cfgs: Vec<String>,
libs: Vec<SearchPath>,
cg: CodegenOptions,
externs: Externs,
should_panic: bool,
no_run: bool,
as_test_harness: bool,
compile_fail: bool,
mut error_codes: Vec<String>,
opts: &TestOptions,
maybe_sysroot: Option<PathBuf>,
linker: Option<PathBuf>,
edition: Edition,
persist_doctests: Option<PathBuf>,
) -> Result<(), TestFailure> {
let (test, line_offset) = match panic::catch_unwind(|| {
make_test(test, Some(cratename), as_test_harness, opts)
}) {
@@ -306,44 +338,43 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,

match (compile_result, compile_fail) {
(Ok(()), true) => {
panic!("test compiled while it wasn't supposed to")
return Err(TestFailure::UnexpectedCompilePass);
}
(Ok(()), false) => {}
(Err(_), true) => {
if error_codes.len() > 0 {
if !error_codes.is_empty() {
let out = String::from_utf8(data.lock().unwrap().to_vec()).unwrap();
error_codes.retain(|err| !out.contains(err));

if !error_codes.is_empty() {
return Err(TestFailure::MissingErrorCodes(error_codes));
}
}
}
(Err(_), false) => {
panic!("couldn't compile the test")
return Err(TestFailure::CompileError);
}
}

if error_codes.len() > 0 {
panic!("Some expected error codes were not found: {:?}", error_codes);
if no_run {
return Ok(());
}

if no_run { return }

// Run the code!
let mut cmd = Command::new(output_file);

match cmd.output() {
Err(e) => panic!("couldn't run the test: {}{}", e,
if e.kind() == io::ErrorKind::PermissionDenied {
" - maybe your tempdir is mounted with noexec?"
} else { "" }),
Err(e) => return Err(TestFailure::ExecutionError(e)),
Ok(out) => {
if should_panic && out.status.success() {
panic!("test executable succeeded when it should have failed");
return Err(TestFailure::UnexpectedRunPass);
} else if !should_panic && !out.status.success() {
panic!("test executable failed:\n{}\n{}\n",
str::from_utf8(&out.stdout).unwrap_or(""),
str::from_utf8(&out.stderr).unwrap_or(""));
return Err(TestFailure::ExecutionFailure(out));
}
}
}

Ok(())
}

/// Transforms a test into code that can be compiled into a Rust binary, and returns the number of
@@ -708,7 +739,7 @@ impl Tester for Collector {
allow_fail: config.allow_fail,
},
testfn: testing::DynTestFn(box move || {
run_test(
let res = run_test(
&test,
&cratename,
&filename,
@@ -727,7 +758,65 @@ impl Tester for Collector {
linker,
edition,
persist_doctests
)
);

if let Err(err) = res {
match err {
TestFailure::CompileError => {
eprint!("Couldn't compile the test.");
}
TestFailure::UnexpectedCompilePass => {
eprint!("Test compiled successfully, but it's marked `compile_fail`.");
}
TestFailure::UnexpectedRunPass => {
eprint!("Test executable succeeded, but it's marked `should_panic`.");
}
TestFailure::MissingErrorCodes(codes) => {
eprint!("Some expected error codes were not found: {:?}", codes);
}
TestFailure::ExecutionError(err) => {
eprint!("Couldn't run the test: {}", err);
if err.kind() == io::ErrorKind::PermissionDenied {
eprint!(" - maybe your tempdir is mounted with noexec?");
}
}
TestFailure::ExecutionFailure(out) => {
let reason = if let Some(code) = out.status.code() {
format!("exit code: {}", code)
} else {
String::from("terminated by signal")
};

eprintln!("Test executable failed ({}).", reason);

// FIXME(#12309): An unfortunate side-effect of capturing the test
// executable's output is that the relative ordering between the test's
// stdout and stderr is lost. However, this is better than the
// alternative: if the test executable inherited the parent's I/O
// handles the output wouldn't be captured at all, even on success.
//
// The ordering could be preserved if the test process' stderr was
// redirected to stdout, but that functionality does not exist in the
// standard library, so it may not be portable enough.
let stdout = str::from_utf8(&out.stdout).unwrap_or_default();
let stderr = str::from_utf8(&out.stderr).unwrap_or_default();

if !stdout.is_empty() || !stderr.is_empty() {
eprintln!();

if !stdout.is_empty() {
eprintln!("stdout:\n{}", stdout);
}

if !stderr.is_empty() {
eprintln!("stderr:\n{}", stderr);
}
}
}
}

panic::resume_unwind(box ());
}
}),
});
}
@@ -0,0 +1,12 @@
// Issue #51162: A failed doctest was not printing its stdout/stderr
// FIXME: if/when the output of the test harness can be tested on its own, this test should be
// adapted to use that, and that normalize line can go away

// compile-flags:--test
// normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
// failure-status: 101

/// ```compile_fail
/// println!("Hello");
/// ```
pub struct Foo;
@@ -0,0 +1,14 @@

running 1 test
test $DIR/failed-doctest-compile-fail.rs - Foo (line 9) ... FAILED

failures:

---- $DIR/failed-doctest-compile-fail.rs - Foo (line 9) stdout ----
Test compiled successfully, but it's marked `compile_fail`.

failures:
$DIR/failed-doctest-compile-fail.rs - Foo (line 9)

test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out

@@ -0,0 +1,12 @@
// Issue #51162: A failed doctest was not printing its stdout/stderr
// FIXME: if/when the output of the test harness can be tested on its own, this test should be
// adapted to use that, and that normalize line can go away

// compile-flags:--test
// normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
// failure-status: 101

/// ```compile_fail,E0004
/// let x: () = 5i32;
/// ```
pub struct Foo;
@@ -0,0 +1,26 @@

running 1 test
test $DIR/failed-doctest-missing-codes.rs - Foo (line 9) ... FAILED

failures:

---- $DIR/failed-doctest-missing-codes.rs - Foo (line 9) stdout ----
error[E0308]: mismatched types
--> $DIR/failed-doctest-missing-codes.rs:10:13
|
3 | let x: () = 5i32;
| ^^^^ expected (), found i32
|
= note: expected type `()`
found type `i32`

error: aborting due to previous error

For more information about this error, try `rustc --explain E0308`.
Some expected error codes were not found: ["E0004"]

failures:
$DIR/failed-doctest-missing-codes.rs - Foo (line 9)

test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out

@@ -5,10 +5,13 @@
// compile-flags:--test
// normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
// failure-status: 101
// rustc-env:RUST_BACKTRACE=0

// doctest fails at runtime
/// ```
/// println!("stdout 1");
/// eprintln!("stderr 1");
/// println!("stdout 2");
/// eprintln!("stderr 2");
/// panic!("oh no");
/// ```
pub struct SomeStruct;
@@ -1,35 +1,39 @@

running 2 tests
test $DIR/failed-doctest-output.rs - OtherStruct (line 17) ... FAILED
test $DIR/failed-doctest-output.rs - SomeStruct (line 11) ... FAILED
test $DIR/failed-doctest-output.rs - OtherStruct (line 20) ... FAILED
test $DIR/failed-doctest-output.rs - SomeStruct (line 10) ... FAILED

failures:

---- $DIR/failed-doctest-output.rs - OtherStruct (line 17) stdout ----
---- $DIR/failed-doctest-output.rs - OtherStruct (line 20) stdout ----
error[E0425]: cannot find value `no` in this scope
--> $DIR/failed-doctest-output.rs:18:1
--> $DIR/failed-doctest-output.rs:21:1
|
3 | no
| ^^ not found in this scope

error: aborting due to previous error

For more information about this error, try `rustc --explain E0425`.
thread '$DIR/failed-doctest-output.rs - OtherStruct (line 17)' panicked at 'couldn't compile the test', src/librustdoc/test.rs:319:13
Couldn't compile the test.
---- $DIR/failed-doctest-output.rs - SomeStruct (line 10) stdout ----
Test executable failed (exit code: 101).

stdout:
stdout 1
stdout 2

stderr:
stderr 1
stderr 2
thread 'main' panicked at 'oh no', $DIR/failed-doctest-output.rs:7:1
note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace.

---- $DIR/failed-doctest-output.rs - SomeStruct (line 11) stdout ----
thread '$DIR/failed-doctest-output.rs - SomeStruct (line 11)' panicked at 'test executable failed:

thread 'main' panicked at 'oh no', $DIR/failed-doctest-output.rs:3:1
note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace.

', src/librustdoc/test.rs:341:17


failures:
$DIR/failed-doctest-output.rs - OtherStruct (line 17)
$DIR/failed-doctest-output.rs - SomeStruct (line 11)
$DIR/failed-doctest-output.rs - OtherStruct (line 20)
$DIR/failed-doctest-output.rs - SomeStruct (line 10)

test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out

@@ -0,0 +1,12 @@
// Issue #51162: A failed doctest was not printing its stdout/stderr
// FIXME: if/when the output of the test harness can be tested on its own, this test should be
// adapted to use that, and that normalize line can go away

// compile-flags:--test
// normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
// failure-status: 101

/// ```should_panic
/// println!("Hello, world!");
/// ```
pub struct Foo;
@@ -0,0 +1,14 @@

running 1 test
test $DIR/failed-doctest-should-panic.rs - Foo (line 9) ... FAILED

failures:

---- $DIR/failed-doctest-should-panic.rs - Foo (line 9) stdout ----
Test executable succeeded, but it's marked `should_panic`.

failures:
$DIR/failed-doctest-should-panic.rs - Foo (line 9)

test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out

@@ -13,9 +13,7 @@ error: unterminated double quote string

error: aborting due to previous error

thread '$DIR/unparseable-doc-test.rs - foo (line 6)' panicked at 'couldn't compile the test', src/librustdoc/test.rs:319:13
note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace.

Couldn't compile the test.

failures:
$DIR/unparseable-doc-test.rs - foo (line 6)

0 comments on commit f1c60e7

Please sign in to comment.
You can’t perform that action at this time.