Auto merge of #108534 - Mark-Simulacrum:compression, r=pietroalbini

Import rust-installer & adjust compression settings

This brings in rust-lang/rust-installer#123, which enables a larger compression window (8 -> 64MB) amongst other changes to the xz compression settings. The net effect should be smaller compressed tarballs which will decrease bandwidth usage for
static.rust-lang.org, download times, and decompression time.

This comes at the cost of higher baseline requirements for running rustup to use these files, which we believe should be largely acceptable (running rustc is likely to use at least this much memory) but if we get specific reports we may explore options to decrease impact (e.g., using the gzip tarballs automatically in rustup).

To simplify iteration on compression settings this also imports the rust-lang/rust-installer submodule, it is now hosted fully inside rust-lang/rust. Once we land this I'll file a followup to add a note to that repo and we can subsequently archive it.

--

CI times for dist-x86_64-linux builds:

* threads=6, master - 2h 50m
* threads=1, new - 3h 40m
* threads=6, new - 2h 50m

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "src/rust-installer"]
-	path = src/tools/rust-installer
-	url = https://github.com/rust-lang/rust-installer.git
 [submodule "src/doc/nomicon"]
 	path = src/doc/nomicon
 	url = https://github.com/rust-lang/nomicon.git

src/bootstrap/builder.rs

@@ -711,6 +711,7 @@ impl<'a> Builder<'a> {
                 test::RustdocUi,
                 test::RustdocJson,
                 test::HtmlCheck,
+                test::RustInstaller,
                 // Run bootstrap close to the end as it's unlikely to fail
                 test::Bootstrap,
                 // Run run-make last, since these won't pass without make on Windows

src/bootstrap/lib.rs

@@ -482,12 +482,7 @@ impl Build {
 
             // Make sure we update these before gathering metadata so we don't get an error about missing
             // Cargo.toml files.
-            let rust_submodules = [
-                "src/tools/rust-installer",
-                "src/tools/cargo",
-                "library/backtrace",
-                "library/stdarch",
-            ];
+            let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"];
             for s in rust_submodules {
                 build.update_submodule(Path::new(s));
             }

src/bootstrap/test.rs

@@ -2695,3 +2695,58 @@ impl Step for LintDocs {
         });
     }
 }
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustInstaller;
+
+impl Step for RustInstaller {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    /// Ensure the version placeholder replacement tool builds
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("test rust-installer");
+
+        let bootstrap_host = builder.config.build;
+        let compiler = builder.compiler(0, bootstrap_host);
+        let cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolBootstrap,
+            bootstrap_host,
+            "test",
+            "src/tools/rust-installer",
+            SourceType::InTree,
+            &[],
+        );
+        try_run(builder, &mut cargo.into());
+
+        // We currently don't support running the test.sh script outside linux(?) environments.
+        // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a
+        // set of scripts, which will likely allow dropping this if.
+        if bootstrap_host != "x86_64-unknown-linux-gnu" {
+            return;
+        }
+
+        let mut cmd =
+            std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh"));
+        let tmpdir = testdir(builder, compiler.host).join("rust-installer");
+        let _ = std::fs::remove_dir_all(&tmpdir);
+        let _ = std::fs::create_dir_all(&tmpdir);
+        cmd.current_dir(&tmpdir);
+        cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target"));
+        cmd.env("CARGO", &builder.initial_cargo);
+        cmd.env("RUSTC", &builder.initial_rustc);
+        cmd.env("TMP_DIR", &tmpdir);
+        try_run(builder, &mut cmd);
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/rust-installer")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self);
+    }
+}

src/tools/rust-installer/.gitignore

@@ -0,0 +1,5 @@
+*~
+tmp
+target/
+**/*.rs.bk
+Cargo.lock

src/tools/rust-installer/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "installer"
+version = "0.0.0"
+edition = "2018"
+
+[[bin]]
+doc = false
+name = "rust-installer"
+path = "src/main.rs"
+
+[dependencies]
+anyhow = "1.0.19"
+flate2 = "1.0.1"
+rayon = "1.0"
+tar = "0.4.13"
+walkdir = "2"
+xz2 = "0.1.4"
+num_cpus = "1"
+remove_dir_all = "0.5"
+
+[dependencies.clap]
+features = ["derive"]
+version = "3.1"
+
+[target."cfg(windows)".dependencies]
+lazy_static = "1"
+winapi = { version = "0.3", features = ["errhandlingapi", "handleapi", "ioapiset", "winerror", "winioctl", "winnt"] }

src/tools/rust-installer/README.md

@@ -0,0 +1,71 @@
+[![Build Status](https://travis-ci.org/rust-lang/rust-installer.svg?branch=master)](https://travis-ci.org/rust-lang/rust-installer)
+
+A generator for the install.sh script commonly used to install Rust in
+Unix environments. It is used By Rust, Cargo, and is intended to be
+used by a future combined installer of Rust + Cargo.
+
+# Usage
+
+```
+./gen-installer.sh --product-name=Rust \
+                   --rel-manifest-dir=rustlib \
+                   --success-message=Rust-is-ready-to-roll. \
+                   --image-dir=./install-image \
+                   --work-dir=./temp \
+                   --output-dir=./dist \
+                   --non-installed-overlay=./overlay \
+                   --package-name=rustc-nightly-i686-apple-darwin \
+                   --component-name=rustc \
+                   --legacy-manifest-dirs=rustlib \
+                   --bulk-dirs=share/doc
+```
+
+Or, to just generate the script.
+
+```
+./gen-install-script.sh --product-name=Rust \
+                        --rel-manifest-dir=rustlib \
+                        --success-message=Rust-is-ready-to-roll. \
+                        --output-script=install.sh \
+                        --legacy-manifest-dirs=rustlib
+```
+
+*Note: the dashes in `success-message` are converted to spaces. The
+script's argument handling is broken with spaces.*
+
+To combine installers.
+
+```
+./combine-installers.sh --product-name=Rust \
+                        --rel-manifest-dir=rustlib \
+                        --success-message=Rust-is-ready-to-roll. \
+                        --work-dir=./temp \
+                        --output-dir=./dist \
+                        --non-installed-overlay=./overlay \
+                        --package-name=rustc-nightly-i686-apple-darwin \
+                        --legacy-manifest-dirs=rustlib \
+                        --input-tarballs=./rustc.tar.gz,cargo.tar.gz
+```
+
+# Future work
+
+* Make install.sh not have to be customized, pull it's data from a
+  config file.
+* Be more resiliant to installation failures, particularly if the disk
+  is full.
+* Pre-install and post-uninstall scripts.
+* Allow components to depend on or contradict other components.
+* Sanity check that expected destination dirs (bin, lib, share exist)?
+* Add --docdir flag. Is there a standard name for this?
+* Remove empty directories on uninstall.
+* Detect mismatches in --prefix, --mandir, etc. in follow-on
+  installs/uninstalls.
+* Fix argument handling for spaces.
+* Add --bindir.
+
+# License
+
+This software is distributed under the terms of both the MIT license
+and/or the Apache License (Version 2.0), at your option.
+
+See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for details.

src/tools/rust-installer/combine-installers.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ue
+
+# Prints the absolute path of a directory to stdout
+abs_path() {
+    local path="$1"
+    # Unset CDPATH because it causes havok: it makes the destination unpredictable
+    # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null
+    # for good measure.
+    (unset CDPATH && cd "$path" > /dev/null && pwd)
+}
+
+src_dir="$(abs_path $(dirname "$0"))"
+$CARGO run --manifest-path="$src_dir/Cargo.toml" -- combine "$@"

src/tools/rust-installer/gen-install-script.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ue
+
+# Prints the absolute path of a directory to stdout
+abs_path() {
+    local path="$1"
+    # Unset CDPATH because it causes havok: it makes the destination unpredictable
+    # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null
+    # for good measure.
+    (unset CDPATH && cd "$path" > /dev/null && pwd)
+}
+
+src_dir="$(abs_path $(dirname "$0"))"
+cargo run --manifest-path="$src_dir/Cargo.toml" -- script "$@"

src/tools/rust-installer/gen-installer.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ue
+
+# Prints the absolute path of a directory to stdout
+abs_path() {
+    local path="$1"
+    # Unset CDPATH because it causes havok: it makes the destination unpredictable
+    # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null
+    # for good measure.
+    (unset CDPATH && cd "$path" > /dev/null && pwd)
+}
+
+src_dir="$(abs_path $(dirname "$0"))"
+$CARGO run --manifest-path="$src_dir/Cargo.toml" -- generate "$@"