[disabled] Remove unused files from the release

src/build_manifest.rs

@@ -1,13 +1,14 @@
-use crate::{config::Channel, Context};
+use crate::Context;
 use anyhow::{Context as _, Error};
 use std::{
+    collections::HashSet,
     fs::File,
     io::BufReader,
     path::{Path, PathBuf},
     process::Command,
 };
 use tar::Archive;
-use tempfile::NamedTempFile;
+use tempfile::{NamedTempFile, TempDir};
 use xz2::read::XzDecoder;
 
 pub(crate) struct BuildManifest<'a> {
@@ -19,10 +20,7 @@ pub(crate) struct BuildManifest<'a> {
 impl<'a> BuildManifest<'a> {
     pub(crate) fn new(builder: &'a Context) -> Self {
         // Precalculate paths used later.
-        let release = match builder.config.channel {
-            Channel::Stable => builder.current_version.clone().unwrap(),
-            channel => channel.to_string(),
-        };
+        let release = builder.config.channel.release_name(builder);
         let tarball_name = format!("build-manifest-{}-{}", release, crate::TARGET);
         let tarball_path = builder.dl_dir().join(format!("{}.tar.xz", tarball_name));
 
@@ -37,12 +35,15 @@ impl<'a> BuildManifest<'a> {
         self.tarball_path.is_file()
     }
 
-    pub(crate) fn run(&self) -> Result<(), Error> {
+    pub(crate) fn run(&self) -> Result<Execution, Error> {
         let config = &self.builder.config;
         let bin = self
             .extract()
             .context("failed to extract build-manifest from the tarball")?;
 
+        let metadata_dir = TempDir::new()?;
+        let shipped_files_path = metadata_dir.path().join("shipped-files.txt");
+
         println!("running build-manifest...");
         let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir);
         // build-manifest <input-dir> <output-dir> <date> <upload-addr> <channel>
@@ -52,11 +53,12 @@ impl<'a> BuildManifest<'a> {
             .arg(&self.builder.date)
             .arg(upload_addr)
             .arg(config.channel.to_string())
+            .env("BUILD_MANIFEST_SHIPPED_FILES_PATH", &shipped_files_path)
             .status()
             .context("failed to execute build-manifest")?;
 
         if status.success() {
-            Ok(())
+            Execution::new(&shipped_files_path)
         } else {
             anyhow::bail!("build-manifest failed with status {:?}", status);
         }
@@ -82,3 +84,27 @@ impl<'a> BuildManifest<'a> {
         Ok(bin)
     }
 }
+
+pub(crate) struct Execution {
+    pub(crate) shipped_files: Option<HashSet<PathBuf>>,
+}
+
+impl Execution {
+    fn new(shipped_files_path: &Path) -> Result<Self, Error> {
+        // Once https://github.com/rust-lang/rust/pull/78196 reaches stable we can assume the
+        // "shipped files" file is always generated, and we can remove the Option<_>.
+        let shipped_files = if shipped_files_path.is_file() {
+            Some(
+                std::fs::read_to_string(shipped_files_path)?
+                    .lines()
+                    .filter(|line| !line.trim().is_empty())
+                    .map(PathBuf::from)
+                    .collect(),
+            )
+        } else {
+            None
+        };
+
+        Ok(Execution { shipped_files })
+    }
+}

src/config.rs

@@ -1,4 +1,5 @@
-use anyhow::{Context, Error};
+use crate::Context;
+use anyhow::{Context as _, Error};
 use std::env::VarError;
 use std::str::FromStr;
 
@@ -11,6 +12,16 @@ pub(crate) enum Channel {
     Nightly,
 }
 
+impl Channel {
+    pub(crate) fn release_name(&self, ctx: &Context) -> String {
+        if *self == Channel::Stable {
+            ctx.current_version.clone().unwrap()
+        } else {
+            self.to_string()
+        }
+    }
+}
+
 impl FromStr for Channel {
     type Err = Error;
 
@@ -70,6 +81,9 @@ pub(crate) struct Config {
     /// Whether to allow multiple releases on the same channel in the same day or not.
     pub(crate) allow_multiple_today: bool,
 
+    /// Whether to allow the work-in-progress pruning code for this release.
+    pub(crate) wip_prune_unused_files: bool,
+
     /// The compression level to use when recompressing tarballs with gzip.
     pub(crate) gzip_compression_level: u32,
     /// Custom sha of the commit to release, instead of the latest commit in the channel's branch.
@@ -105,6 +119,7 @@ impl Config {
             upload_addr: require_env("UPLOAD_ADDR")?,
             upload_bucket: require_env("UPLOAD_BUCKET")?,
             upload_dir: require_env("UPLOAD_DIR")?,
+            wip_prune_unused_files: bool_env("WIP_PRUNE_UNUSED_FILES")?,
         })
     }
 }

src/main.rs

@@ -2,12 +2,12 @@ mod build_manifest;
 mod config;
 mod sign;
 
-use std::env;
 use std::fs::{self, File, OpenOptions};
 use std::io::{self, Read};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::Instant;
+use std::{collections::HashSet, env};
 
 use anyhow::Error;
 use build_manifest::BuildManifest;
@@ -22,6 +22,14 @@ use crate::config::{Channel, Config};
 
 const TARGET: &str = env!("TARGET");
 
+/// List of files that should not be pruned even if they're not referenced in the manifest.
+const AVOID_PRUNING_UNUSED_FILES: &[&str] = &[
+    // Source code tarballs, which are not distributed through rustup and thus are mistakenly
+    // marked as "unused" by build-manifest.
+    "rustc-{release}-src.tar.gz",
+    "rustc-{release}-src.tar.xz",
+];
+
 struct Context {
     work: PathBuf,
     handle: Easy,
@@ -177,7 +185,14 @@ impl Context {
         let build_manifest = BuildManifest::new(self);
         if build_manifest.exists() {
             // Generate the channel manifest
-            build_manifest.run()?;
+            let execution = build_manifest.run()?;
+
+            if self.config.wip_prune_unused_files {
+                // Removes files that we are not shipping from the files we're about to upload.
+                if let Some(shipped_files) = &execution.shipped_files {
+                    self.prune_unused_files(&shipped_files)?;
+                }
+            }
 
             // Generate checksums and sign all the files we're about to ship.
             let signer = Signer::new(&self.config)?;
@@ -459,6 +474,28 @@ upload-addr = \"{}/{}\"
         Ok(())
     }
 
+    fn prune_unused_files(&self, shipped_files: &HashSet<PathBuf>) -> Result<(), Error> {
+        let release = self.config.channel.release_name(self);
+        let allowed_files = AVOID_PRUNING_UNUSED_FILES
+            .iter()
+            .map(|pattern| pattern.replace("{release}", &release).into())
+            .collect::<HashSet<PathBuf>>();
+
+        for entry in std::fs::read_dir(self.dl_dir())? {
+            let entry = entry?;
+
+            if let Some(name) = entry.path().file_name() {
+                let name = Path::new(name);
+                if !allowed_files.contains(name) && !shipped_files.contains(name) {
+                    std::fs::remove_file(entry.path())?;
+                    println!("pruned unused file {}", name.display());
+                }
+            }
+        }
+
+        Ok(())
+    }
+
     fn publish_archive(&mut self) -> Result<(), Error> {
         let bucket = &self.config.upload_bucket;
         let dir = &self.config.upload_dir;