Add support for rustfix coverage tracking

src/common.rs

@@ -215,6 +215,11 @@ pub struct Config {
     /// where to find the remote test client process, if we're using it
     pub remote_test_client: Option<PathBuf>,
 
+    /// If true, this will generate a coverage file with UI test files that run `MachineApplicable`
+    /// diagnostics but are missing `run-rustfix` annotations. The generated coverage file is
+    /// created in `/<build_base>/rustfix_missing_coverage.txt`
+    pub rustfix_coverage: bool,
+
     // Configuration for various run-make tests frobbing things like C compilers
     // or querying about various LLVM component information.
     pub cc: String,
@@ -359,6 +364,7 @@ impl Default for Config {
             host: platform.clone(),
             #[cfg(feature = "norustc")]
             host: env!("HOST").to_string(),
+            rustfix_coverage: false,
             gdb: None,
             gdb_version: None,
             gdb_native_rust: false,

src/lib.rs

@@ -75,6 +75,18 @@ pub fn run_tests(config: &Config) {
         env::set_var("RUST_TEST_TASKS", "1");
     }
 
+    // If we want to collect rustfix coverage information,
+    // we first make sure that the coverage file does not exist.
+    // It will be created later on.
+    if config.rustfix_coverage {
+        let mut coverage_file_path = config.build_base.clone();
+        coverage_file_path.push("rustfix_missing_coverage.txt");
+        if coverage_file_path.exists() {
+            if let Err(e) = fs::remove_file(&coverage_file_path) {
+                panic!("Could not delete {} due to {}", coverage_file_path.display(), e)
+            }
+        }
+    }
     let opts = test_opts(config);
     let tests = make_tests(config);
     // sadly osx needs some file descriptor limits raised for running tests in

src/runtest.rs

@@ -26,7 +26,7 @@ use std::collections::HashMap;
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
-use std::fs::{self, File, create_dir_all};
+use std::fs::{self, File, create_dir_all, OpenOptions};
 use std::fmt;
 use std::io::prelude::*;
 use std::io::{self, BufReader};
@@ -2268,6 +2268,36 @@ actual:\n\
         errors += self.compare_output(UI_STDERR, &normalized_stderr, &expected_stderr);
 
 
+        if self.config.rustfix_coverage {
+            // Find out which tests have `MachineApplicable` suggestions but are missing
+            // `run-rustfix` or `run-rustfix-only-machine-applicable` headers.
+            //
+            // This will return an empty `Vec` in case the executed test file has a
+            // `compile-flags: --error-format=xxxx` header with a value other than `json`.
+            let suggestions = get_suggestions_from_json(
+                &proc_res.stderr,
+                &HashSet::new(),
+                Filter::MachineApplicableOnly
+            ).unwrap_or_default();
+            if suggestions.len() > 0
+                && !self.props.run_rustfix
+                && !self.props.rustfix_only_machine_applicable {
+                    let mut coverage_file_path = self.config.build_base.clone();
+                    coverage_file_path.push("rustfix_missing_coverage.txt");
+                    debug!("coverage_file_path: {}", coverage_file_path.display());
+
+                    let mut file = OpenOptions::new()
+                        .create(true)
+                        .append(true)
+                        .open(coverage_file_path.as_path())
+                        .expect("could not create or open file");
+
+                    if let Err(_) = writeln!(file, "{}", self.testpaths.file.display()) {
+                        panic!("couldn't write to {}", coverage_file_path.display());
+                    }
+            }
+        }
+
         if self.props.run_rustfix {
             // Apply suggestions from lints to the code itself
             let unfixed_code = self