UI tests extract the regular output from the 'rendered' field in json

src/tools/compiletest/src/json.rs

@@ -57,6 +57,25 @@ struct DiagnosticCode {
     explanation: Option<String>,
 }
 
+pub fn extract_rendered(output: &str, proc_res: &ProcRes) -> String {
+    output.lines()
+        .filter_map(|line| if line.starts_with('{') {
+            match json::decode::<Diagnostic>(line) {
+                Ok(diagnostic) => diagnostic.rendered,
+                Err(error) => {
+                    proc_res.fatal(Some(&format!("failed to decode compiler output as json: \
+                                                `{}`\noutput: {}\nline: {}",
+                                                error,
+                                                line,
+                                                output)));
+                }
+            }
+        } else {
+            None
+        })
+        .collect()
+}
+
 pub fn parse_output(file_name: &str, output: &str, proc_res: &ProcRes) -> Vec<Error> {
     output.lines()
         .flat_map(|line| parse_line(file_name, line, output, proc_res))

src/tools/compiletest/src/runtest.rs

@@ -1403,6 +1403,11 @@ actual:\n\
                     rustc.args(&["--error-format", "json"]);
                 }
             }
+            Ui => {
+                if !self.props.compile_flags.iter().any(|s| s.starts_with("--error-format")) {
+                    rustc.args(&["--error-format", "json"]);
+                }
+            }
             MirOpt => {
                 rustc.args(&[
                     "-Zdump-mir=all",
@@ -1427,7 +1432,6 @@ actual:\n\
             Codegen |
             Rustdoc |
             RunMake |
-            Ui |
             CodegenUnits => {
                 // do not use JSON output
             }
@@ -2211,7 +2215,12 @@ actual:\n\
     }
 
     fn run_ui_test(&self) {
-        let proc_res = self.compile_test();
+        // if the user specified a format in the ui test
+        // print the output to the stderr file, otherwise extract
+        // the rendered error messages from json and print them
+        let explicit = self.props.compile_flags.iter().any(|s| s.starts_with("--error-format"));
+
+        let mut proc_res = self.compile_test();
 
         let expected_stderr_path = self.expected_output_path("stderr");
         let expected_stderr = self.load_expected_output(&expected_stderr_path);
@@ -2220,14 +2229,24 @@ actual:\n\
         let expected_stdout = self.load_expected_output(&expected_stdout_path);
 
         let normalized_stdout =
-            self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout);
+            self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout, explicit);
+
+        let stderr = if explicit {
+            proc_res.stderr.clone()
+        } else {
+            json::extract_rendered(&proc_res.stderr, &proc_res)
+        };
+
         let normalized_stderr =
-            self.normalize_output(&proc_res.stderr, &self.props.normalize_stderr);
+            self.normalize_output(&stderr, &self.props.normalize_stderr, explicit);
 
         let mut errors = 0;
         errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
         errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
 
+        // rewrite the output to the human readable one (shown in case of errors)
+        proc_res.stderr = normalized_stderr;
+
         if errors > 0 {
             println!("To update references, run this command from build directory:");
             let relative_path_to_file =
@@ -2421,11 +2440,13 @@ actual:\n\
         mir_dump_dir
     }
 
-    fn normalize_output(&self, output: &str, custom_rules: &[(String, String)]) -> String {
+    fn normalize_output(
+        &self,
+        output: &str,
+        custom_rules: &[(String, String)],
+        json: bool,
+    ) -> String {
         let parent_dir = self.testpaths.file.parent().unwrap();
-        let cflags = self.props.compile_flags.join(" ");
-        let json = cflags.contains("--error-format json") ||
-                   cflags.contains("--error-format pretty-json");
         let parent_dir_str = if json {
             parent_dir.display().to_string().replace("\\", "\\\\")
         } else {