Rollup merge of rust-lang#7298 - flip1995:ci-switch-to-monster, r=xFr…

…ednet,flip1995

Switch CI to new metadata collection

r? `@xFrednet`

Things we have to keep in mind:

- This removes the template files and the scripts used for deployment from the checkout. This was added in rust-lang#5517. I don't think we ever needed those there. Not sure though.
- ~~As a result, we can't remove the python scripts yet. We have to wait until this hits a stable Clippy release.~~ I'll just break the next stable deploy and do it by hand once.
- This should be merged together with rust-lang#7279. Me and `@xFrednet` will coordinate the switch
- ...?

I still have to try out some things:

- [x] Is it worth caching? Yes
- [x] ~~Is it worth to do a release build?~~ Nope
- [x] Does it actually work? With a few changes, yes
- [ ] ...?

changelog: Clippy now uses a lint to generate its documentation 🎉

.github/deploy.sh

@@ -8,13 +8,12 @@ rm -rf out/master/ || exit 0
 echo "Making the docs for master"
 mkdir out/master/
 cp util/gh-pages/index.html out/master
-python3 ./util/export.py out/master/lints.json
+cp util/gh-pages/lints.json out/master
 
 if [[ -n $TAG_NAME ]]; then
   echo "Save the doc for the current tag ($TAG_NAME) and point stable/ to it"
-  cp -r out/master "out/$TAG_NAME"
-  rm -f out/stable
-  ln -s "$TAG_NAME" out/stable
+  cp -Tr out/master "out/$TAG_NAME"
+  ln -sf "$TAG_NAME" out/stable
 fi
 
 if [[ $BETA = "true" ]]; then
@@ -28,8 +27,8 @@ cp util/gh-pages/versions.html out/index.html
 echo "Making the versions.json file"
 python3 ./util/versions.py out
 
-cd out
 # Now let's go have some fun with the cloned repo
+cd out
 git config user.name "GHA CI"
 git config user.email "gha@ci.invalid"
 

.github/workflows/deploy.yml

@@ -39,10 +39,23 @@ jobs:
       if: github.ref == 'refs/heads/beta'
       run: echo "BETA=true" >> $GITHUB_ENV
 
-    - name: Use scripts and templates from master branch
+    # We need to check out all files that (transitively) depend on the
+    # structure of the gh-pages branch, so that we're able to change that
+    # structure without breaking the deployment.
+    - name: Use deploy files from master branch
       run: |
         git fetch --no-tags --prune --depth=1 origin master
-        git checkout origin/master -- .github/deploy.sh util/gh-pages/ util/*.py
+        git checkout origin/master -- .github/deploy.sh util/versions.py util/gh-pages/versions.html
+
+    # Generate lockfile for caching to avoid build problems with cached deps
+    - name: cargo generate-lockfile
+      run: cargo generate-lockfile
+
+    - name: Cache
+      uses: Swatinem/rust-cache@v1.3.0
+
+    - name: cargo collect-metadata
+      run: cargo collect-metadata
 
     - name: Deploy
       run: |

clippy_dev/src/serve.rs

@@ -15,8 +15,8 @@ pub fn run(port: u16, lint: Option<&str>) -> ! {
 
     loop {
         if mtime("util/gh-pages/lints.json") < mtime("clippy_lints/src") {
-            Command::new("python3")
-                .arg("util/export.py")
+            Command::new("cargo")
+                .arg("collect-metadata")
                 .spawn()
                 .unwrap()
                 .wait()

clippy_lints/src/utils/internal_lints/metadata_collector.rs

@@ -32,7 +32,7 @@ use clippy_utils::{
 };
 
 /// This is the output file of the lint collector.
-const OUTPUT_FILE: &str = "../util/gh-pages/metadata_collection.json";
+const OUTPUT_FILE: &str = "../util/gh-pages/lints.json";
 /// These lints are excluded from the export.
 const BLACK_LISTED_LINTS: [&str; 3] = ["lint_author", "deep_code_inspection", "internal_metadata_collector"];
 /// These groups will be ignored by the lint group matcher. This is useful for collections like

tests/dogfood.rs

@@ -183,7 +183,7 @@ fn run_metadata_collection_lint() {
     use std::time::SystemTime;
 
     // Setup for validation
-    let metadata_output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("util/gh-pages/metadata_collection.json");
+    let metadata_output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("util/gh-pages/lints.json");
     let start_time = SystemTime::now();
 
     // Run collection as is

util/versions.py

@@ -3,8 +3,8 @@
 import json
 import os
 import sys
-
-from lintlib import log
+import logging as log
+log.basicConfig(level=log.INFO, format='%(levelname)s: %(message)s')
 
 
 def key(v):
@@ -26,7 +26,7 @@ def key(v):
 
 def main():
     if len(sys.argv) < 2:
-        print("Error: specify output directory")
+        log.error("specify output directory")
         return
 
     outdir = sys.argv[1]