Merge pull request #793 from Skgland/note-spurious-retry-list-entries

mention amount of spurious entries on retry-regressed-list.txt …

Author: Mark-Simulacrum

src/report/mod.rs

@@ -398,6 +398,9 @@ pub fn gen<DB: ReadResults, W: ReportWriter + Display>(
     Ok(res)
 }
 
+pub(crate) const SPURIOUS_RETRY: &[Comparison] =
+    &[Comparison::SpuriousRegressed, Comparison::PrepareFail];
+
 /// Generates a list of regressed crate names that can be passed to crater via
 /// `crates=list:...` to retry those.
 fn gen_retry_list(res: &RawTestResults) -> String {
@@ -409,9 +412,7 @@ fn gen_retry_list(res: &RawTestResults) -> String {
         .crates
         .iter()
         .filter(|crate_res| {
-            crate_res.res == Comparison::Regressed
-                || crate_res.res == Comparison::SpuriousRegressed
-                || crate_res.res == Comparison::PrepareFail
+            crate_res.res == Comparison::Regressed || SPURIOUS_RETRY.contains(&crate_res.res)
         })
         .map(|crate_res| &crate_res.krate);
 

src/server/reports.rs

@@ -1,6 +1,6 @@
 use crate::experiments::{Experiment, Status};
 use crate::prelude::*;
-use crate::report::{self, Comparison, TestResults};
+use crate::report::{self, Comparison, TestResults, SPURIOUS_RETRY};
 use crate::results::DatabaseDB;
 use crate::server::messages::{Label, Message};
 use crate::server::{Data, GithubData};
@@ -99,14 +99,21 @@ fn reports_thread(data: &Data, github_data: Option<&GithubData>) -> Fallible<()>
                     .public_url
                     .replace("{bucket}", &data.tokens.reports_bucket.bucket);
                 let report_url = format!("{base_url}/{name}/index.html");
+                let retry_regressed_list_url =
+                    format!("{base_url}/{name}/retry-regressed-list.txt");
 
                 ex.set_status(&data.db, Status::Completed)?;
                 ex.set_report_url(&data.db, &report_url)?;
+
                 info!("report for the experiment {name} generated successfully!");
 
-                let (regressed, fixed) = (
+                let (regressed, fixed, spurious_retry) = (
                     res.info.get(&Comparison::Regressed).unwrap_or(&0),
                     res.info.get(&Comparison::Fixed).unwrap_or(&0),
+                    SPURIOUS_RETRY
+                        .iter()
+                        .flat_map(|comp| res.info.get(comp))
+                        .sum::<u32>(),
                 );
 
                 if let Some(github_data) = github_data {
@@ -122,6 +129,7 @@ fn reports_thread(data: &Data, github_data: Option<&GithubData>) -> Fallible<()>
                                     res.info.values().sum::<u32>(),
                                 ),
                             )
+                            .line("bar_chart", format!(" {spurious_retry} spurious results on the [retry-regessed-list.txt]({retry_regressed_list_url}), consider a retry[^1] if this is a significant amount.\n[^1]: re-run the experiment with `crates={retry_regressed_list_url}`"))
                             .line(
                                 "newspaper",
                                 format!("[Open the summary report]({report_url})."),