Merge pull request #899 from googlefonts/crater-ci

Checkpoint 'crater in CI' code.

fontc_crater/Cargo.toml

@@ -10,8 +10,10 @@ readme = "README.md"
 [dependencies]
 fontc = { version = "0.0.1", path = "../fontc" }
 
-google-fonts-sources = "0.1.0"
+google-fonts-sources = "0.2.0"
+maud = "0.26.0"
 
+chrono.workspace = true
 rayon.workspace = true
 write-fonts.workspace = true
 serde.workspace = true

fontc_crater/README.md

@@ -10,6 +10,12 @@ fonts.
 $ cargo run --release -p=fontc_crater -- compile FONT_CACHE --fonts-repo GOOGLE/FONTS  -o results.json
 ```
 
+or, to run ttx_diff (comparing with fontmake)
+
+```sh
+$ cargo run --release -p=fontc_crater -- diff FONT_CACHE --fonts-repo GOOGLE/FONTS  -o results.json
+```
+
 This is a binary for executing font compilation (and possibly other tasks) in
 bulk.
 
@@ -36,6 +42,16 @@ You can generate a report from the saved json by passing it back to
 $ cargo run -p fontc_crater -- report results.json
 ```
 
+## CI
+
+This binary is also run in CI. In that case, the execution is managed by a
+script in the [`fontc_crater` repo][crater-repo] and results are posted to
+[github pages][crater-results].
+
 [google-fonts-sources]: https://github.com/googlefonts/google-fonts-sources
 [google/fonts]: https://github.com/google/fonts
 [rust-lang/crater]: https://github.com/rust-lang/crater
+[crater-repo]: https://github.com/googlefonts/fontc_crater
+[crater-results]: https://googlefonts.github.io/fontc_crater/
+
+

fontc_crater/resources/ci.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# this script is responsible for running crater. It is expected to be called
+# by `fetch_and_run.sh`.
+
+echo "this is the script that will actually run CI"
+# these are set by the script that calls us
+echo "input is $FONTC_CRATER_INPUT, results written to $FONTC_CRATER_RESULTS"
+cargo run -p fontc_crater --release -- ci $FONTC_CRATER_INPUT --out $FONTC_CRATER_RESULTS

fontc_crater/resources/style.css

@@ -0,0 +1,59 @@
+/*style sheet bundled up in the html generated by crater*/
+body {
+  font-family: sans-serif;
+}
+
+table {
+  table-layout: fixed;
+  width: 100%;
+  border-collapse: collapse;
+  border: 1px solid #bbb;
+}
+
+tbody td, th {
+  text-align: left;
+}
+
+th {
+  padding: 5px 5px;
+  border-bottom: 1px solid #bbb;
+}
+
+td {
+  padding: 2px 5px;
+}
+
+.better {
+  color: green;
+  font-size: 0.8em;
+}
+
+.worse {
+  color: red;
+  font-size: 0.8em;
+}
+
+td.rev {
+  font-family: monospace;
+}
+
+thead th:nth-child(1) {
+  width: 17%
+}
+
+thead th:nth-child(4) {
+  width: 8%
+}
+
+thead th:nth-child(5) {
+  width: 9%
+}
+
+thead th:nth-child(7) {
+  width: 9%
+}
+
+#explain {
+  padding-bottom: 10px;
+  color: #888;
+}

fontc_crater/src/args.rs

@@ -16,6 +16,7 @@ pub(super) enum Commands {
     Compile(RunArgs),
     Diff(RunArgs),
     Report(ReportArgs),
+    Ci(CiArgs),
 }
 
 #[derive(Debug, PartialEq, clap::Args)]
@@ -37,6 +38,20 @@ pub(super) struct RunArgs {
     pub(super) n_fonts: Option<usize>,
 }
 
+#[derive(Debug, PartialEq, clap::Args)]
+pub(super) struct CiArgs {
+    /// Path to a json list of repos + revs to run.
+    pub(super) to_run: PathBuf,
+    /// Directory where results are written.
+    ///
+    /// This should be consistent between runs.
+    #[arg(short = 'o', long = "out")]
+    pub(super) out_dir: PathBuf,
+    /// only generate html (for the provided out_dir)
+    #[arg(long)]
+    pub(super) html_only: bool,
+}
+
 #[derive(Debug, PartialEq, clap::Args)]
 pub(super) struct ReportArgs {
     pub(super) json_path: PathBuf,

fontc_crater/src/ci.rs

@@ -0,0 +1,124 @@
+//! Running in CI.
+//!
+//! This handles the whole CI workflow.
+//!
+//! Unlike a normal run, this is expecting to have preexisting results, and to
+//! generate a fuller report that includes comparison with past runs.
+
+use std::path::{Path, PathBuf};
+
+use chrono::{DateTime, Utc};
+use google_fonts_sources::RepoInfo;
+
+use crate::{
+    args::CiArgs,
+    error::Error,
+    ttx_diff_runner::{DiffError, DiffOutput},
+    Results,
+};
+
+mod html;
+
+static SUMMARY_FILE: &str = "summary.json";
+
+type DiffResults = Results<DiffOutput, DiffError>;
+
+/// A summary of a single CI run
+
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+struct RunSummary {
+    began: DateTime<Utc>,
+    finished: DateTime<Utc>,
+    fontc_rev: String,
+    results_file: PathBuf,
+    // the name of the file listing targets used by this run.
+    // it is intended that when this list is updated, the filename is changed.
+    input_file: PathBuf,
+    stats: super::ttx_diff_runner::Summary,
+}
+
+impl RunSummary {
+    fn try_load_results(&self, target_dir: &Path) -> Option<Result<DiffResults, Error>> {
+        let report_path = target_dir.join(&self.results_file);
+        if !report_path.exists() {
+            return None;
+        }
+        Some(super::try_read_json(&report_path))
+    }
+}
+
+pub(super) fn run_ci(args: &CiArgs) -> Result<(), Error> {
+    if !args.html_only {
+        super::ttx_diff_runner::assert_can_run_script();
+        run_crater_and_save_results(args)?;
+    }
+    html::generate(&args.out_dir)?;
+    // now we want to generate an html report, based on this info.
+    Ok(())
+}
+
+fn run_crater_and_save_results(args: &CiArgs) -> Result<(), Error> {
+    if !args.out_dir.exists() {
+        super::try_create_dir(&args.out_dir)?;
+    }
+
+    let summary_file = args.out_dir.join(SUMMARY_FILE);
+    let mut prev_runs: Vec<RunSummary> = if summary_file.exists() {
+        super::try_read_json(&summary_file)?
+    } else {
+        Default::default()
+    };
+
+    // todo: fontc_repo should be checked out by us, and have a known path
+    let fontc_rev = super::get_git_rev(None).unwrap();
+    if prev_runs
+        .last()
+        .map(|prev| prev.fontc_rev == fontc_rev)
+        .unwrap_or(false)
+    {
+        // eventually we will want an argument to force rerunning, which we'll
+        // use when we update fontmake or the input data set
+        eprintln!("fontc rev is unchange from last run, skipping");
+        return Ok(());
+    }
+
+    let inputs: Vec<RepoInfo> = super::try_read_json(&args.to_run)?;
+    let out_file = result_path_for_current_date();
+    let out_path = args.out_dir.join(&out_file);
+    // for now we are going to be cloning each repo freshly
+    let cache_dir = tempfile::tempdir().unwrap();
+
+    let began = Utc::now();
+    let results = super::run_all(
+        &inputs,
+        cache_dir.path(),
+        super::ttx_diff_runner::run_ttx_diff,
+    )?;
+    let finished = Utc::now();
+
+    super::try_write_json(&results, &out_path)?;
+    let summary = super::ttx_diff_runner::Summary::new(&results);
+    let input_file = args
+        .to_run
+        .file_name()
+        .map(PathBuf::from)
+        .unwrap_or_else(|| args.to_run.clone());
+
+    let summary = RunSummary {
+        began,
+        finished,
+        fontc_rev,
+        results_file: out_file.into(),
+        input_file,
+        stats: summary,
+    };
+
+    prev_runs.push(summary);
+    super::try_write_json(&prev_runs, &summary_file)
+}
+
+fn result_path_for_current_date() -> String {
+    let now = chrono::Utc::now();
+    let timestamp = now.format("%Y-%m-%d-%H%M%S");
+    format!("{timestamp}.json")
+}