ci(metrics): Run measurement functions in parallel

feat(xtask): Split metrics function
rust-lang · Jun 11, 2023 · b909f4e · b909f4e
1 parent d567091
commit b909f4e
Show file tree

Hide file tree

Showing 3 changed files with 291 additions and 46 deletions.
diff --git a/.github/workflows/metrics.yaml b/.github/workflows/metrics.yaml
@@ -11,20 +11,234 @@ env:
   RUSTUP_MAX_RETRIES: 10
 
 jobs:
-  metrics:
-    if: github.repository == 'rust-lang/rust-analyzer'
+  setup_cargo:
     runs-on: ubuntu-latest
+    steps:
+      - name: Install Rust toolchain
+        run: |
+          rustup update --no-self-update stable
+          rustup component add rustfmt rust-src
+      - name: Cache cargo
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ github.sha }}
+
+  build_metrics:
+    runs-on: ubuntu-latest
+    needs: setup_cargo
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
 
-    - name: Install Rust toolchain
-      run: |
-        rustup update --no-self-update stable
-        rustup component add rustfmt rust-src
+    - name: Restore cargo cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+        key: ${{ runner.os }}-cargo-${{ github.sha }}
+
+
+    - name: Collect build metrics
+      run: cargo xtask metrics build
+
+    - name: Cache target
+      uses: actions/cache@v3
+      with:
+        path: target/
+        key: ${{ runner.os }}-target-${{ github.sha }}
+
+    - name: Upload build metrics
+      uses: actions/upload-artifact@v3
+      with:
+        name: build-${{ github.sha }}
+        path: target/build.json
+        if-no-files-found: error
+
+  self_metrics:
+    runs-on: ubuntu-latest
+    needs: [setup_cargo, build_metrics]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Restore cargo cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+        key: ${{ runner.os }}-cargo-${{ github.sha }}
+
+    - name: Restore target cache
+      uses: actions/cache@v3
+      with:
+        path: target/
+        key: ${{ runner.os }}-target-${{ github.sha }}
 
-    - name: Collect metrics
-      run: cargo xtask metrics
-      env:
-        METRICS_TOKEN: ${{ secrets.METRICS_TOKEN }}
+    - name: Collect build metrics
+      run: cargo xtask metrics self
+
+    - name: Upload build metrics
+      uses: actions/upload-artifact@v3
+      with:
+        name: self-${{ github.sha }}
+        path: target/self.json
+        if-no-files-found: error
+
+  ripgrep_metrics:
+    runs-on: ubuntu-latest
+    needs: [setup_cargo, build_metrics]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Restore cargo cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+        key: ${{ runner.os }}-cargo-${{ github.sha }}
+
+    - name: Restore target cache
+      uses: actions/cache@v3
+      with:
+        path: target/
+        key: ${{ runner.os }}-target-${{ github.sha }}
+
+    - name: Collect build metrics
+      run: cargo xtask metrics ripgrep
+
+    - name: Upload ripgrep metrics
+      uses: actions/upload-artifact@v3
+      with:
+        name: ripgrep-${{ github.sha }}
+        path: target/ripgrep.json
+        if-no-files-found: error
+
+  webrender_metrics:
+    runs-on: ubuntu-latest
+    needs: [setup_cargo, build_metrics]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Restore cargo cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+        key: ${{ runner.os }}-cargo-${{ github.sha }}
+
+    - name: Restore target cache
+      uses: actions/cache@v3
+      with:
+        path: target/
+        key: ${{ runner.os }}-target-${{ github.sha }}
+
+    - name: Collect webrender metrics
+      run: cargo xtask metrics webrender
+
+    - name: Upload webrender metrics
+      uses: actions/upload-artifact@v3
+      with:
+        name: webrender-${{ github.sha }}
+        path: target/webrender.json
+        if-no-files-found: error
+
+  diesel_metrics:
+    runs-on: ubuntu-latest
+    needs: [setup_cargo, build_metrics]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Restore cargo cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+        key: ${{ runner.os }}-cargo-${{ github.sha }}
+
+    - name: Restore target cache
+      uses: actions/cache@v3
+      with:
+        path: target/
+        key: ${{ runner.os }}-target-${{ github.sha }}
+
+    - name: Collect build metrics
+      run: cargo xtask metrics diesel
+
+    - name: Upload build metrics
+      uses: actions/upload-artifact@v3
+      with:
+        name: diesel-${{ github.sha }}
+        path: target/diesel.json
+        if-no-files-found: error
+
+
+
+  generate_final_metrics:
+    runs-on: ubuntu-latest
+    needs: [build_metrics, self_metrics, ripgrep_metrics, webrender_metrics, diesel_metrics]
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Download build metrics
+      uses: actions/download-artifact@v3
+      with:
+        name: build-${{ github.sha }}
+
+    - name: Download self metrics
+      uses: actions/download-artifact@v3
+      with:
+        name: self-${{ github.sha }}
+
+    - name: Download ripgrep metrics
+      uses: actions/download-artifact@v3
+      with:
+        name: ripgrep-${{ github.sha }}
+
+    - name: Download webrender metrics
+      uses: actions/download-artifact@v3
+      with:
+        name: webrender-${{ github.sha }}
+
+    - name: Download diesel metrics
+      uses: actions/download-artifact@v3
+      with:
+        name: diesel-${{ github.sha }}
+
+    - name: Combine json
+      run: |
+        git clone --depth 1 https://$METRICS_TOKEN@github.com/nanthR/metrics.git
+        jq -s ".[0] * .[1] * .[2] * .[3] * .[4]" build.json self.json ripgrep.json webrender.json diesel.json -c >> metrics/metrics.json
+        git -C metrics add .
+        git -C metrics -c user.name=Bot -c user.email=dummy@example.com commit --message 📈
+        git -C metrics push origin master
+    env:
+      METRICS_TOKEN: ${{ secrets.METRICS_TOKEN }}
diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs
@@ -1,5 +1,7 @@
 #![allow(unreachable_pub)]
 
+use std::str::FromStr;
+
 use crate::install::{ClientOpt, Malloc, ServerOpt};
 
 xflags::xflags! {
@@ -42,7 +44,7 @@ xflags::xflags! {
             required changelog: String
         }
         cmd metrics {
-            optional --dry-run
+            optional measurement_type: MeasurementType
         }
         /// Builds a benchmark version of rust-analyzer and puts it into `./target`.
         cmd bb {
@@ -105,9 +107,32 @@ pub struct PublishReleaseNotes {
     pub dry_run: bool,
 }
 
+#[derive(Debug)]
+pub enum MeasurementType {
+    Build,
+    AnalyseSelf,
+    AnalyseRipgrep,
+    AnalyseWebRender,
+    AnalyseDiesel,
+}
+
+impl FromStr for MeasurementType {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "build" => Ok(Self::Build),
+            "self" => Ok(Self::AnalyseSelf),
+            "ripgrep" => Ok(Self::AnalyseRipgrep),
+            "webrender" => Ok(Self::AnalyseWebRender),
+            "diesel" => Ok(Self::AnalyseDiesel),
+            _ => Err("Invalid option".to_string()),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct Metrics {
-    pub dry_run: bool,
+    pub measurement_type: Option<MeasurementType>,
 }
 
 #[derive(Debug)]

diff --git a/xtask/src/metrics.rs b/xtask/src/metrics.rs
@@ -9,16 +9,13 @@ use std::{
 use anyhow::{bail, format_err};
 use xshell::{cmd, Shell};
 
-use crate::flags;
+use crate::flags::{self, MeasurementType};
 
 type Unit = String;
 
 impl flags::Metrics {
     pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
         let mut metrics = Metrics::new(sh)?;
-        if !self.dry_run {
-            sh.remove_path("./target/release")?;
-        }
         if !Path::new("./target/rustc-perf").exists() {
             sh.create_dir("./target/rustc-perf")?;
             cmd!(sh, "git clone https://github.com/rust-lang/rustc-perf.git ./target/rustc-perf")
@@ -32,38 +29,47 @@ impl flags::Metrics {
 
         let _env = sh.push_env("RA_METRICS", "1");
 
-        {
-            // https://github.com/rust-lang/rust-analyzer/issues/9997
-            let _d = sh.push_dir("target/rustc-perf/collector/benchmarks/webrender");
-            cmd!(sh, "cargo update -p url --precise 1.6.1").run()?;
-        }
-        metrics.measure_build(sh)?;
-        metrics.measure_analysis_stats_self(sh)?;
-        metrics.measure_analysis_stats(sh, "ripgrep")?;
-        metrics.measure_analysis_stats(sh, "webrender")?;
-        metrics.measure_analysis_stats(sh, "diesel/diesel")?;
-
-        if !self.dry_run {
-            let _d = sh.push_dir("target");
-            let metrics_token = env::var("METRICS_TOKEN").unwrap();
-            cmd!(
-                sh,
-                "git clone --depth 1 https://{metrics_token}@github.com/rust-analyzer/metrics.git"
-            )
-            .run()?;
-
-            {
-                let mut file =
-                    fs::File::options().append(true).open("target/metrics/metrics.json")?;
-                writeln!(file, "{}", metrics.json())?;
+        let filename = match self.measurement_type {
+            Some(ms) => match ms {
+                MeasurementType::Build => {
+                    metrics.measure_build(sh)?;
+                    "build.json"
+                }
+                MeasurementType::AnalyseSelf => {
+                    metrics.measure_analysis_stats_self(sh)?;
+                    "self.json"
+                }
+                MeasurementType::AnalyseRipgrep => {
+                    metrics.measure_analysis_stats(sh, "ripgrep")?;
+                    "ripgrep.json"
+                }
+                MeasurementType::AnalyseWebRender => {
+                    {
+                        // https://github.com/rust-lang/rust-analyzer/issues/9997
+                        let _d = sh.push_dir("target/rustc-perf/collector/benchmarks/webrender");
+                        cmd!(sh, "cargo update -p url --precise 1.6.1").run()?;
+                    }
+                    metrics.measure_analysis_stats(sh, "webrender")?;
+                    "webrender.json"
+                }
+                MeasurementType::AnalyseDiesel => {
+                    metrics.measure_analysis_stats(sh, "diesel/diesel")?;
+                    "diesel.json"
+                }
+            },
+            None => {
+                metrics.measure_build(sh)?;
+                metrics.measure_analysis_stats_self(sh)?;
+                metrics.measure_analysis_stats(sh, "ripgrep")?;
+                metrics.measure_analysis_stats(sh, "webrender")?;
+                metrics.measure_analysis_stats(sh, "diesel/diesel")?;
+                "all.json"
             }
+        };
 
-            let _d = sh.push_dir("metrics");
-            cmd!(sh, "git add .").run()?;
-            cmd!(sh, "git -c user.name=Bot -c user.email=dummy@example.com commit --message 📈")
-                .run()?;
-            cmd!(sh, "git push origin master").run()?;
-        }
+        let mut file =
+            fs::File::options().write(true).create(true).open(format!("target/{}", filename))?;
+        writeln!(file, "{}", metrics.json())?;
         eprintln!("{metrics:#?}");
         Ok(())
     }