From e733ff797faeaaca1ede731e483e59b9e83c764b Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Mon, 27 Feb 2023 10:53:13 -0500 Subject: [PATCH 1/4] Bump rust-installer This brings in rust-lang/rust-installer#123, which enables a larger compression window (8 -> 64MB) amongst other changes to the xz compression settings. The net effect should be smaller compressed tarballs which will decrease bandwidth usage for static.rust-lang.org, download times, and decompression time. This comes at the cost of higher baseline requirements for running rustup to use these files, which we believe should be largely acceptable (running rustc is likely to use at least this much memory) but if we get specific reports we may explore options to decrease impact (e.g., using the gzip tarballs automatically in rustup). --- src/tools/rust-installer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rust-installer b/src/tools/rust-installer index 9981e4d1ea6ac..31b4e313213df 160000 --- a/src/tools/rust-installer +++ b/src/tools/rust-installer @@ -1 +1 @@ -Subproject commit 9981e4d1ea6ac0992ff21be5514d4230dc77548b +Subproject commit 31b4e313213dfaf62b2dd13a4da8176990929526 From 8d9cef4709db316c35d78d3cd1d13d970edbd57c Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Tue, 28 Feb 2023 21:02:17 -0500 Subject: [PATCH 2/4] Directly import rust-installer submodule This moves the rust-installer code to be directly hosted in rust-lang/rust, since it's not used elsewhere and this makes it easier to make and review changes without needing a separate upstream commit. --- .gitmodules | 3 - src/tools/rust-installer | 1 - src/tools/rust-installer/.gitignore | 5 + src/tools/rust-installer/Cargo.toml | 28 + src/tools/rust-installer/README.md | 71 + .../rust-installer/combine-installers.sh | 24 + .../rust-installer/gen-install-script.sh | 24 + src/tools/rust-installer/gen-installer.sh | 24 + src/tools/rust-installer/install-template.sh | 1005 ++++++++++++ src/tools/rust-installer/make-tarballs.sh | 24 + .../rust-installer/rust-installer-version | 1 + src/tools/rust-installer/src/combiner.rs | 161 ++ src/tools/rust-installer/src/compression.rs | 203 +++ src/tools/rust-installer/src/generator.rs | 178 +++ src/tools/rust-installer/src/lib.rs | 17 + src/tools/rust-installer/src/main.rs | 27 + .../rust-installer/src/remove_dir_all.rs | 860 +++++++++++ src/tools/rust-installer/src/scripter.rs | 68 + src/tools/rust-installer/src/tarballer.rs | 143 ++ src/tools/rust-installer/src/util.rs | 156 ++ src/tools/rust-installer/test.sh | 1342 +++++++++++++++++ .../test/image-docdir1/share/doc/rust/README | 1 + .../image-docdir1/share/doc/rust/rustdocs.txt | 1 + .../test/image-docdir2/share/doc/cargo/README | 1 + .../share/doc/cargo/cargodocs.txt | 1 + .../rust-installer/test/image1/bin/bad-bin | 1 + .../rust-installer/test/image1/bin/program | 1 + .../rust-installer/test/image1/bin/program2 | 1 + .../test/image1/dir-to-install/foo | 0 .../test/image1/dir-to-not-install/foo | 0 .../test/image1/something-to-install | 0 .../test/image1/something-to-not-install | 0 .../rust-installer/test/image2/bin/oldprogram | 1 + .../test/image2/dir-to-install/bar | 0 .../test/image2/something-to-install | 0 .../rust-installer/test/image3/bin/cargo | 1 + src/tools/rust-installer/test/image4/baz | 0 .../test/image4/dir-to-install/qux/bar | 0 .../test/image5/dir-to-install/foo | 0 39 files changed, 4370 insertions(+), 4 deletions(-) delete mode 160000 src/tools/rust-installer create mode 100644 src/tools/rust-installer/.gitignore create mode 100644 src/tools/rust-installer/Cargo.toml create mode 100644 src/tools/rust-installer/README.md create mode 100755 src/tools/rust-installer/combine-installers.sh create mode 100755 src/tools/rust-installer/gen-install-script.sh create mode 100755 src/tools/rust-installer/gen-installer.sh create mode 100644 src/tools/rust-installer/install-template.sh create mode 100755 src/tools/rust-installer/make-tarballs.sh create mode 100644 src/tools/rust-installer/rust-installer-version create mode 100644 src/tools/rust-installer/src/combiner.rs create mode 100644 src/tools/rust-installer/src/compression.rs create mode 100644 src/tools/rust-installer/src/generator.rs create mode 100644 src/tools/rust-installer/src/lib.rs create mode 100644 src/tools/rust-installer/src/main.rs create mode 100644 src/tools/rust-installer/src/remove_dir_all.rs create mode 100644 src/tools/rust-installer/src/scripter.rs create mode 100644 src/tools/rust-installer/src/tarballer.rs create mode 100644 src/tools/rust-installer/src/util.rs create mode 100755 src/tools/rust-installer/test.sh create mode 100644 src/tools/rust-installer/test/image-docdir1/share/doc/rust/README create mode 100644 src/tools/rust-installer/test/image-docdir1/share/doc/rust/rustdocs.txt create mode 100644 src/tools/rust-installer/test/image-docdir2/share/doc/cargo/README create mode 100644 src/tools/rust-installer/test/image-docdir2/share/doc/cargo/cargodocs.txt create mode 100644 src/tools/rust-installer/test/image1/bin/bad-bin create mode 100755 src/tools/rust-installer/test/image1/bin/program create mode 100755 src/tools/rust-installer/test/image1/bin/program2 create mode 100644 src/tools/rust-installer/test/image1/dir-to-install/foo create mode 100644 src/tools/rust-installer/test/image1/dir-to-not-install/foo create mode 100644 src/tools/rust-installer/test/image1/something-to-install create mode 100644 src/tools/rust-installer/test/image1/something-to-not-install create mode 100755 src/tools/rust-installer/test/image2/bin/oldprogram create mode 100644 src/tools/rust-installer/test/image2/dir-to-install/bar create mode 100644 src/tools/rust-installer/test/image2/something-to-install create mode 100755 src/tools/rust-installer/test/image3/bin/cargo create mode 100644 src/tools/rust-installer/test/image4/baz create mode 100644 src/tools/rust-installer/test/image4/dir-to-install/qux/bar create mode 100644 src/tools/rust-installer/test/image5/dir-to-install/foo diff --git a/.gitmodules b/.gitmodules index 4011a6fa6b95b..e79f2f089c1ed 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "src/rust-installer"] - path = src/tools/rust-installer - url = https://github.com/rust-lang/rust-installer.git [submodule "src/doc/nomicon"] path = src/doc/nomicon url = https://github.com/rust-lang/nomicon.git diff --git a/src/tools/rust-installer b/src/tools/rust-installer deleted file mode 160000 index 31b4e313213df..0000000000000 --- a/src/tools/rust-installer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 31b4e313213dfaf62b2dd13a4da8176990929526 diff --git a/src/tools/rust-installer/.gitignore b/src/tools/rust-installer/.gitignore new file mode 100644 index 0000000000000..fb017f484b159 --- /dev/null +++ b/src/tools/rust-installer/.gitignore @@ -0,0 +1,5 @@ +*~ +tmp +target/ +**/*.rs.bk +Cargo.lock diff --git a/src/tools/rust-installer/Cargo.toml b/src/tools/rust-installer/Cargo.toml new file mode 100644 index 0000000000000..38b81a1baacd8 --- /dev/null +++ b/src/tools/rust-installer/Cargo.toml @@ -0,0 +1,28 @@ +[package] +authors = ["The Rust Project Developers"] +name = "installer" +version = "0.0.0" +edition = "2018" + +[[bin]] +doc = false +name = "rust-installer" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.19" +flate2 = "1.0.1" +rayon = "1.0" +tar = "0.4.13" +walkdir = "2" +xz2 = "0.1.4" +num_cpus = "1" +remove_dir_all = "0.5" + +[dependencies.clap] +features = ["derive"] +version = "3.1" + +[target."cfg(windows)".dependencies] +lazy_static = "1" +winapi = { version = "0.3", features = ["errhandlingapi", "handleapi", "ioapiset", "winerror", "winioctl", "winnt"] } diff --git a/src/tools/rust-installer/README.md b/src/tools/rust-installer/README.md new file mode 100644 index 0000000000000..99d8e5ca4cf1c --- /dev/null +++ b/src/tools/rust-installer/README.md @@ -0,0 +1,71 @@ +[![Build Status](https://travis-ci.org/rust-lang/rust-installer.svg?branch=master)](https://travis-ci.org/rust-lang/rust-installer) + +A generator for the install.sh script commonly used to install Rust in +Unix environments. It is used By Rust, Cargo, and is intended to be +used by a future combined installer of Rust + Cargo. + +# Usage + +``` +./gen-installer.sh --product-name=Rust \ + --rel-manifest-dir=rustlib \ + --success-message=Rust-is-ready-to-roll. \ + --image-dir=./install-image \ + --work-dir=./temp \ + --output-dir=./dist \ + --non-installed-overlay=./overlay \ + --package-name=rustc-nightly-i686-apple-darwin \ + --component-name=rustc \ + --legacy-manifest-dirs=rustlib \ + --bulk-dirs=share/doc +``` + +Or, to just generate the script. + +``` +./gen-install-script.sh --product-name=Rust \ + --rel-manifest-dir=rustlib \ + --success-message=Rust-is-ready-to-roll. \ + --output-script=install.sh \ + --legacy-manifest-dirs=rustlib +``` + +*Note: the dashes in `success-message` are converted to spaces. The +script's argument handling is broken with spaces.* + +To combine installers. + +``` +./combine-installers.sh --product-name=Rust \ + --rel-manifest-dir=rustlib \ + --success-message=Rust-is-ready-to-roll. \ + --work-dir=./temp \ + --output-dir=./dist \ + --non-installed-overlay=./overlay \ + --package-name=rustc-nightly-i686-apple-darwin \ + --legacy-manifest-dirs=rustlib \ + --input-tarballs=./rustc.tar.gz,cargo.tar.gz +``` + +# Future work + +* Make install.sh not have to be customized, pull it's data from a + config file. +* Be more resiliant to installation failures, particularly if the disk + is full. +* Pre-install and post-uninstall scripts. +* Allow components to depend on or contradict other components. +* Sanity check that expected destination dirs (bin, lib, share exist)? +* Add --docdir flag. Is there a standard name for this? +* Remove empty directories on uninstall. +* Detect mismatches in --prefix, --mandir, etc. in follow-on + installs/uninstalls. +* Fix argument handling for spaces. +* Add --bindir. + +# License + +This software is distributed under the terms of both the MIT license +and/or the Apache License (Version 2.0), at your option. + +See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for details. diff --git a/src/tools/rust-installer/combine-installers.sh b/src/tools/rust-installer/combine-installers.sh new file mode 100755 index 0000000000000..4931c34ddc27f --- /dev/null +++ b/src/tools/rust-installer/combine-installers.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2014 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ue + +# Prints the absolute path of a directory to stdout +abs_path() { + local path="$1" + # Unset CDPATH because it causes havok: it makes the destination unpredictable + # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null + # for good measure. + (unset CDPATH && cd "$path" > /dev/null && pwd) +} + +src_dir="$(abs_path $(dirname "$0"))" +cargo run --manifest-path="$src_dir/Cargo.toml" -- combine "$@" diff --git a/src/tools/rust-installer/gen-install-script.sh b/src/tools/rust-installer/gen-install-script.sh new file mode 100755 index 0000000000000..b4559d147addd --- /dev/null +++ b/src/tools/rust-installer/gen-install-script.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2014 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ue + +# Prints the absolute path of a directory to stdout +abs_path() { + local path="$1" + # Unset CDPATH because it causes havok: it makes the destination unpredictable + # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null + # for good measure. + (unset CDPATH && cd "$path" > /dev/null && pwd) +} + +src_dir="$(abs_path $(dirname "$0"))" +cargo run --manifest-path="$src_dir/Cargo.toml" -- script "$@" diff --git a/src/tools/rust-installer/gen-installer.sh b/src/tools/rust-installer/gen-installer.sh new file mode 100755 index 0000000000000..198cfe7425534 --- /dev/null +++ b/src/tools/rust-installer/gen-installer.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2014 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ue + +# Prints the absolute path of a directory to stdout +abs_path() { + local path="$1" + # Unset CDPATH because it causes havok: it makes the destination unpredictable + # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null + # for good measure. + (unset CDPATH && cd "$path" > /dev/null && pwd) +} + +src_dir="$(abs_path $(dirname "$0"))" +cargo run --manifest-path="$src_dir/Cargo.toml" -- generate "$@" diff --git a/src/tools/rust-installer/install-template.sh b/src/tools/rust-installer/install-template.sh new file mode 100644 index 0000000000000..7790541a4201a --- /dev/null +++ b/src/tools/rust-installer/install-template.sh @@ -0,0 +1,1005 @@ +#!/bin/bash +# Copyright 2014 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +# No undefined variables +set -u + +init_logging() { + local _abs_libdir="$1" + local _logfile="$_abs_libdir/$TEMPLATE_REL_MANIFEST_DIR/install.log" + rm -f "$_logfile" + need_ok "failed to remove old installation log" + touch "$_logfile" + need_ok "failed to create installation log" + LOGFILE="$_logfile" +} + +log_line() { + local _line="$1" + + if [ -n "${LOGFILE-}" -a -e "${LOGFILE-}" ]; then + echo "$_line" >> "$LOGFILE" + # Ignore errors, which may happen e.g. after the manifest dir is deleted + fi +} + +msg() { + local _line="install: ${1-}" + echo "$_line" + log_line "$_line" +} + +verbose_msg() { + if [ -n "${CFG_VERBOSE-}" ]; then + msg "${1-}" + else + log_line "install: ${1-}" + fi +} + +step_msg() { + msg + msg "$1" + msg +} + +verbose_step_msg() { + if [ -n "${CFG_VERBOSE-}" ]; then + msg + msg "$1" + msg + else + log_line "" + log_line "install: $1" + log_line "" + fi +} + +warn() { + local _line="install: WARNING: $1" + echo "$_line" >&2 + log_line "$_line" +} + +err() { + local _line="install: error: $1" + echo "$_line" >&2 + log_line "$_line" + exit 1 +} + +# A non-user error that is likely to result in a corrupted install +critical_err() { + local _line="install: error: $1. see logs at '${LOGFILE-}'" + echo "$_line" >&2 + log_line "$_line" + exit 1 +} + +need_ok() { + if [ $? -ne 0 ] + then + err "$1" + fi +} + +critical_need_ok() { + if [ $? -ne 0 ] + then + critical_err "$1" + fi +} + +want_ok() { + if [ $? -ne 0 ]; then + warn "$1" + fi +} + +assert_nz() { + if [ -z "$1" ]; then err "assert_nz $2"; fi +} + +need_cmd() { + if command -v $1 >/dev/null 2>&1 + then verbose_msg "found $1" + else err "need $1" + fi +} + +run() { + local _line="\$ $*" + "$@" + local _retval=$? + log_line "$_line" + return $_retval +} + +write_to_file() { + local _msg="$1" + local _file="$2" + local _line="$ echo \"$_msg\" > \"$_file\"" + echo "$_msg" > "$_file" + local _retval=$? + log_line "$_line" + return $_retval +} + +append_to_file() { + local _msg="$1" + local _file="$2" + local _line="$ echo \"$_msg\" >> \"$_file\"" + echo "$_msg" >> "$_file" + local _retval=$? + log_line "$_line" + return $_retval +} + +make_dir_recursive() { + local _dir="$1" + local _line="$ umask 022 && mkdir -p \"$_dir\"" + umask 022 && mkdir -p "$_dir" + local _retval=$? + log_line "$_line" + return $_retval +} + +putvar() { + local t + local tlen + eval t=\$$1 + eval tlen=\${#$1} +} + +valopt() { + VAL_OPTIONS="$VAL_OPTIONS $1" + + local op=$1 + local default=$2 + shift + shift + local doc="$*" + if [ $HELP -eq 0 ] + then + local uop=$(echo $op | tr 'a-z-' 'A-Z_') + local v="CFG_${uop}" + eval $v="$default" + for arg in $CFG_ARGS + do + if echo "$arg" | grep -q -- "--$op=" + then + local val=$(echo "$arg" | cut -f2 -d=) + eval $v=$val + fi + done + putvar $v + else + if [ -z "$default" ] + then + default="" + fi + op="${op}=[${default}]" + printf " --%-30s %s\n" "$op" "$doc" + fi +} + +opt() { + BOOL_OPTIONS="$BOOL_OPTIONS $1" + + local op=$1 + local default=$2 + shift + shift + local doc="$*" + local flag="" + + if [ $default -eq 0 ] + then + flag="enable" + else + flag="disable" + doc="don't $doc" + fi + + if [ $HELP -eq 0 ] + then + for arg in $CFG_ARGS + do + if [ "$arg" = "--${flag}-${op}" ] + then + op=$(echo $op | tr 'a-z-' 'A-Z_') + flag=$(echo $flag | tr 'a-z' 'A-Z') + local v="CFG_${flag}_${op}" + eval $v=1 + putvar $v + fi + done + else + if [ ! -z "${META-}" ] + then + op="$op=<$META>" + fi + printf " --%-30s %s\n" "$flag-$op" "$doc" + fi +} + +flag() { + BOOL_OPTIONS="$BOOL_OPTIONS $1" + + local op=$1 + shift + local doc="$*" + + if [ $HELP -eq 0 ] + then + for arg in $CFG_ARGS + do + if [ "$arg" = "--${op}" ] + then + op=$(echo $op | tr 'a-z-' 'A-Z_') + local v="CFG_${op}" + eval $v=1 + putvar $v + fi + done + else + if [ ! -z "${META-}" ] + then + op="$op=<$META>" + fi + printf " --%-30s %s\n" "$op" "$doc" + fi +} + +validate_opt () { + for arg in $CFG_ARGS + do + local is_arg_valid=0 + for option in $BOOL_OPTIONS + do + if test --disable-$option = $arg + then + is_arg_valid=1 + fi + if test --enable-$option = $arg + then + is_arg_valid=1 + fi + if test --$option = $arg + then + is_arg_valid=1 + fi + done + for option in $VAL_OPTIONS + do + if echo "$arg" | grep -q -- "--$option=" + then + is_arg_valid=1 + fi + done + if [ "$arg" = "--help" ] + then + echo + echo "No more help available for Configure options," + echo "check the Wiki or join our IRC channel" + break + else + if test $is_arg_valid -eq 0 + then + err "Option '$arg' is not recognized" + fi + fi + done +} + +absolutify() { + local file_path="$1" + local file_path_dirname="$(dirname "$file_path")" + local file_path_basename="$(basename "$file_path")" + local file_abs_path="$(abs_path "$file_path_dirname")" + local file_path="$file_abs_path/$file_path_basename" + # This is the return value + RETVAL="$file_path" +} + +# Prints the absolute path of a directory to stdout +abs_path() { + local path="$1" + # Unset CDPATH because it causes havok: it makes the destination unpredictable + # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null + # for good measure. + (unset CDPATH && cd "$path" > /dev/null && pwd) +} + +uninstall_legacy() { + local _abs_libdir="$1" + + local _uninstalled_something=false + + # Replace commas in legacy manifest list with spaces + _legacy_manifest_dirs=`echo "$TEMPLATE_LEGACY_MANIFEST_DIRS" | sed "s/,/ /g"` + + # Uninstall from legacy manifests + local _md + for _md in $_legacy_manifest_dirs; do + # First, uninstall from the installation prefix. + # Errors are warnings - try to rm everything in the manifest even if some fail. + if [ -f "$_abs_libdir/$_md/manifest" ] + then + + # iterate through installed manifest and remove files + local _p; + while read _p; do + # the installed manifest contains absolute paths + msg "removing legacy file $_p" + if [ -f "$_p" ] + then + run rm -f "$_p" + want_ok "failed to remove $_p" + else + warn "supposedly installed file $_p does not exist!" + fi + done < "$_abs_libdir/$_md/manifest" + + # If we fail to remove $md below, then the + # installed manifest will still be full; the installed manifest + # needs to be empty before install. + msg "removing legacy manifest $_abs_libdir/$_md/manifest" + run rm -f "$_abs_libdir/$_md/manifest" + # For the above reason, this is a hard error + need_ok "failed to remove installed manifest" + + # Remove $template_rel_manifest_dir directory + msg "removing legacy manifest dir $_abs_libdir/$_md" + run rm -R "$_abs_libdir/$_md" + want_ok "failed to remove $_md" + + _uninstalled_something=true + fi + done + + RETVAL="$_uninstalled_something" +} + +uninstall_components() { + local _abs_libdir="$1" + local _dest_prefix="$2" + local _components="$3" + + # We're going to start by uninstalling existing components. This + local _uninstalled_something=false + + # First, try removing any 'legacy' manifests from before + # rust-installer + uninstall_legacy "$_abs_libdir" + assert_nz "$RETVAL", "RETVAL" + if [ "$RETVAL" = true ]; then + _uninstalled_something=true; + fi + + # Load the version of the installed installer + local _installed_version= + if [ -f "$abs_libdir/$TEMPLATE_REL_MANIFEST_DIR/rust-installer-version" ]; then + _installed_version=`cat "$_abs_libdir/$TEMPLATE_REL_MANIFEST_DIR/rust-installer-version"` + + # Sanity check + if [ ! -n "$_installed_version" ]; then critical_err "rust installer version is empty"; fi + fi + + # If there's something installed, then uninstall + if [ -n "$_installed_version" ]; then + # Check the version of the installed installer + case "$_installed_version" in + + # If this is a previous version, then upgrade in place to the + # current version before uninstalling. + 2 ) + # The only change between version 2 -> 3 is that components are placed + # in subdirectories of the installer tarball. There are no changes + # to the installed data format, so nothing to do. + ;; + + # This is the current version. Nothing need to be done except uninstall. + "$TEMPLATE_RUST_INSTALLER_VERSION") + ;; + + # If this is an unknown (future) version then bail. + * ) + echo "The copy of $TEMPLATE_PRODUCT_NAME at $_dest_prefix was installed using an" + echo "unknown version ($_installed_version) of rust-installer." + echo "Uninstall it first with the installer used for the original installation" + echo "before continuing." + exit 1 + ;; + esac + + local _md="$_abs_libdir/$TEMPLATE_REL_MANIFEST_DIR" + local _installed_components="$(cat "$_md/components")" + + # Uninstall (our components only) before reinstalling + local _available_component + for _available_component in $_components; do + local _installed_component + for _installed_component in $_installed_components; do + if [ "$_available_component" = "$_installed_component" ]; then + msg "uninstalling component '$_available_component'" + local _component_manifest="$_md/manifest-$_installed_component" + + # Sanity check: there should be a component manifest + if [ ! -f "$_component_manifest" ]; then + critical_err "installed component '$_installed_component' has no manifest" + fi + + # Iterate through installed component manifest and remove files + local _directive + while read _directive; do + + local _command=`echo $_directive | cut -f1 -d:` + local _file=`echo $_directive | cut -f2 -d:` + + # Sanity checks + if [ ! -n "$_command" ]; then critical_err "malformed installation directive"; fi + if [ ! -n "$_file" ]; then critical_err "malformed installation directive"; fi + + case "$_command" in + file) + verbose_msg "removing file $_file" + if [ -f "$_file" ]; then + run rm -f "$_file" + want_ok "failed to remove $_file" + else + warn "supposedly installed file $_file does not exist!" + fi + ;; + + dir) + verbose_msg "removing directory $_file" + run rm -r "$_file" + want_ok "unable to remove directory $_file" + ;; + + *) + critical_err "unknown installation directive" + ;; + esac + + done < "$_component_manifest" + + # Remove the installed component manifest + verbose_msg "removing component manifest $_component_manifest" + run rm "$_component_manifest" + # This is a hard error because the installation is unrecoverable + critical_need_ok "failed to remove installed manifest for component '$_installed_component'" + + # Update the installed component list + local _modified_components="$(sed "/^$_installed_component\$/d" "$_md/components")" + write_to_file "$_modified_components" "$_md/components" + critical_need_ok "failed to update installed component list" + fi + done + done + + # If there are no remaining components delete the manifest directory, + # but only if we're doing an uninstall - if we're doing an install, + # then leave the manifest directory around to hang onto the logs, + # and any files not managed by the installer. + if [ -n "${CFG_UNINSTALL-}" ]; then + local _remaining_components="$(cat "$_md/components")" + if [ ! -n "$_remaining_components" ]; then + verbose_msg "removing manifest directory $_md" + run rm -r "$_md" + want_ok "failed to remove $_md" + + maybe_unconfigure_ld + fi + fi + + _uninstalled_something=true + fi + + # There's no installed version. If we were asked to uninstall, then that's a problem. + if [ -n "${CFG_UNINSTALL-}" -a "$_uninstalled_something" = false ] + then + err "unable to find installation manifest at $CFG_LIBDIR/$TEMPLATE_REL_MANIFEST_DIR" + fi +} + +install_components() { + local _src_dir="$1" + local _abs_libdir="$2" + local _dest_prefix="$3" + local _components="$4" + + local _component + for _component in $_components; do + + msg "installing component '$_component'" + + # The file name of the manifest we're installing from + local _input_manifest="$_src_dir/$_component/manifest.in" + + # Sanity check: do we have our input manifests? + if [ ! -f "$_input_manifest" ]; then + critical_err "manifest for $_component does not exist at $_input_manifest" + fi + + # The installed manifest directory + local _md="$_abs_libdir/$TEMPLATE_REL_MANIFEST_DIR" + + # The file name of the manifest we're going to create during install + local _installed_manifest="$_md/manifest-$_component" + + # Create the installed manifest, which we will fill in with absolute file paths + touch "$_installed_manifest" + critical_need_ok "failed to create installed manifest" + + # Add this component to the installed component list + append_to_file "$_component" "$_md/components" + critical_need_ok "failed to update components list for $_component" + + # Now install, iterate through the new manifest and copy files + local _directive + while read _directive; do + + local _command=`echo $_directive | cut -f1 -d:` + local _file=`echo $_directive | cut -f2 -d:` + + # Sanity checks + if [ ! -n "$_command" ]; then critical_err "malformed installation directive"; fi + if [ ! -n "$_file" ]; then critical_err "malformed installation directive"; fi + + # Decide the destination of the file + local _file_install_path="$_dest_prefix/$_file" + + if echo "$_file" | grep "^etc/" > /dev/null + then + local _f="$(echo "$_file" | sed 's/^etc\///')" + _file_install_path="$CFG_SYSCONFDIR/$_f" + fi + + if echo "$_file" | grep "^bin/" > /dev/null + then + local _f="$(echo "$_file" | sed 's/^bin\///')" + _file_install_path="$CFG_BINDIR/$_f" + fi + + if echo "$_file" | grep "^lib/" > /dev/null + then + local _f="$(echo "$_file" | sed 's/^lib\///')" + _file_install_path="$CFG_LIBDIR/$_f" + fi + + if echo "$_file" | grep "^share" > /dev/null + then + local _f="$(echo "$_file" | sed 's/^share\///')" + _file_install_path="$CFG_DATADIR/$_f" + fi + + if echo "$_file" | grep "^share/man/" > /dev/null + then + local _f="$(echo "$_file" | sed 's/^share\/man\///')" + _file_install_path="$CFG_MANDIR/$_f" + fi + + # HACK: Try to support overriding --docdir. Paths with the form + # "share/doc/$product/" can be redirected to a single --docdir + # path. If the following detects that --docdir has been specified + # then it will replace everything preceeding the "$product" path + # component. The problem here is that the combined rust installer + # contains two "products": rust and cargo; so the contents of those + # directories will both be dumped into the same directory; and the + # contents of those directories are _not_ disjoint. Since this feature + # is almost entirely to support 'make install' anyway I don't expect + # this problem to be a big deal in practice. + if [ "$CFG_DOCDIR" != "" ] + then + if echo "$_file" | grep "^share/doc/" > /dev/null + then + local _f="$(echo "$_file" | sed 's/^share\/doc\/[^/]*\///')" + _file_install_path="$CFG_DOCDIR/$_f" + fi + fi + + # Make sure there's a directory for it + make_dir_recursive "$(dirname "$_file_install_path")" + critical_need_ok "directory creation failed" + + # Make the path absolute so we can uninstall it later without + # starting from the installation cwd + absolutify "$_file_install_path" + _file_install_path="$RETVAL" + assert_nz "$_file_install_path" "file_install_path" + + case "$_command" in + file ) + + verbose_msg "copying file $_file_install_path" + + maybe_backup_path "$_file_install_path" + + if echo "$_file" | grep "^bin/" > /dev/null || test -x "$_src_dir/$_component/$_file" + then + run cp "$_src_dir/$_component/$_file" "$_file_install_path" + run chmod 755 "$_file_install_path" + else + run cp "$_src_dir/$_component/$_file" "$_file_install_path" + run chmod 644 "$_file_install_path" + fi + critical_need_ok "file creation failed" + + # Update the manifest + append_to_file "file:$_file_install_path" "$_installed_manifest" + critical_need_ok "failed to update manifest" + + ;; + + dir ) + + verbose_msg "copying directory $_file_install_path" + + maybe_backup_path "$_file_install_path" + + run cp -R "$_src_dir/$_component/$_file" "$_file_install_path" + critical_need_ok "failed to copy directory" + + # Set permissions. 0755 for dirs, 644 for files + run chmod -R u+rwX,go+rX,go-w "$_file_install_path" + critical_need_ok "failed to set permissions on directory" + + # Update the manifest + append_to_file "dir:$_file_install_path" "$_installed_manifest" + critical_need_ok "failed to update manifest" + ;; + + *) + critical_err "unknown installation directive" + ;; + esac + done < "$_input_manifest" + + done +} + +maybe_configure_ld() { + local _abs_libdir="$1" + + local _ostype="$(uname -s)" + assert_nz "$_ostype" "ostype" + + if [ "$_ostype" = "Linux" -a ! -n "${CFG_DISABLE_LDCONFIG-}" ]; then + + # Fedora-based systems do not configure the dynamic linker to look + # /usr/local/lib, which is our default installation directory. To + # make things just work, try to put that directory in + # /etc/ld.so.conf.d/rust-installer-v1 so ldconfig picks it up. + # Issue #30. + # + # This will get rm'd when the last component is uninstalled in + # maybe_unconfigure_ld. + if [ "$_abs_libdir" = "/usr/local/lib" -a -d "/etc/ld.so.conf.d" ]; then + echo "$_abs_libdir" > "/etc/ld.so.conf.d/rust-installer-v1-$TEMPLATE_REL_MANIFEST_DIR.conf" + if [ $? -ne 0 ]; then + # This shouldn't happen if we've gotten this far + # installing to /usr/local + warn "failed to update /etc/ld.so.conf.d. this is unexpected" + fi + fi + + verbose_msg "running ldconfig" + if [ -n "${CFG_VERBOSE-}" ]; then + ldconfig + else + ldconfig 2> /dev/null + fi + if [ $? -ne 0 ] + then + warn "failed to run ldconfig. this may happen when not installing as root. run with --verbose to see the error" + fi + fi +} + +maybe_unconfigure_ld() { + local _ostype="$(uname -s)" + assert_nz "$_ostype" "ostype" + + if [ "$_ostype" != "Linux" ]; then + return 0 + fi + + rm "/etc/ld.so.conf.d/rust-installer-v1-$TEMPLATE_REL_MANIFEST_DIR.conf" 2> /dev/null + # Above may fail since that file may not have been created on install +} + +# Doing our own 'install'-like backup that is consistent across platforms +maybe_backup_path() { + local _file_install_path="$1" + + if [ -e "$_file_install_path" ]; then + msg "backing up existing file at $_file_install_path" + run mv -f "$_file_install_path" "$_file_install_path.old" + critical_need_ok "failed to back up $_file_install_path" + fi +} + +install_uninstaller() { + local _src_dir="$1" + local _src_basename="$2" + local _abs_libdir="$3" + + local _uninstaller="$_abs_libdir/$TEMPLATE_REL_MANIFEST_DIR/uninstall.sh" + msg "creating uninstall script at $_uninstaller" + run cp "$_src_dir/$_src_basename" "$_uninstaller" + critical_need_ok "unable to install uninstaller" +} + +do_preflight_sanity_checks() { + local _src_dir="$1" + local _dest_prefix="$2" + + # Sanity check: can we can write to the destination? + verbose_msg "verifying destination is writable" + make_dir_recursive "$CFG_LIBDIR" + need_ok "can't write to destination. consider \`sudo\`." + touch "$CFG_LIBDIR/rust-install-probe" > /dev/null + if [ $? -ne 0 ] + then + err "can't write to destination. consider \`sudo\`." + fi + rm "$CFG_LIBDIR/rust-install-probe" + need_ok "failed to remove install probe" + + # Sanity check: don't install to the directory containing the installer. + # That would surely cause chaos. + verbose_msg "verifying destination is not the same as source" + local _prefix_dir="$(abs_path "$dest_prefix")" + if [ "$_src_dir" = "$_dest_prefix" -a "${CFG_UNINSTALL-}" != 1 ]; then + err "cannot install to same directory as installer" + fi +} + +verbose_msg "looking for install programs" +verbose_msg + +need_cmd mkdir +need_cmd printf +need_cmd cut +need_cmd grep +need_cmd uname +need_cmd tr +need_cmd sed +need_cmd chmod +need_cmd env +need_cmd pwd + +CFG_ARGS="${@:-}" + +HELP=0 +if [ "${1-}" = "--help" ] +then + HELP=1 + shift + echo + echo "Usage: $0 [options]" + echo + echo "Options:" + echo +else + verbose_step_msg "processing arguments" +fi + +OPTIONS="" +BOOL_OPTIONS="" +VAL_OPTIONS="" + +flag uninstall "only uninstall from the installation prefix" +valopt destdir "" "set installation root" +valopt prefix "/usr/local" "set installation prefix" + +# Avoid prepending an extra / to the prefix path if there's no destdir +# NB: CFG vars here are undefined when passing --help +if [ -z "${CFG_DESTDIR-}" ]; then + CFG_DESTDIR_PREFIX="${CFG_PREFIX-}" +else + CFG_DESTDIR_PREFIX="$CFG_DESTDIR/$CFG_PREFIX" +fi + +# NB This isn't quite the same definition as in `configure`. +# just using 'lib' instead of configure's CFG_LIBDIR_RELATIVE +valopt without "" "comma-separated list of components to not install" +valopt components "" "comma-separated list of components to install" +flag list-components "list available components" +valopt sysconfdir "$CFG_DESTDIR_PREFIX/etc" "install system configuration files" +valopt bindir "$CFG_DESTDIR_PREFIX/bin" "install binaries" +valopt libdir "$CFG_DESTDIR_PREFIX/lib" "install libraries" +valopt datadir "$CFG_DESTDIR_PREFIX/share" "install data" +# NB We repeat datadir default value because we don't set CFG_DATADIR in --help +valopt mandir "${CFG_DATADIR-"$CFG_DESTDIR_PREFIX/share"}/man" "install man pages in PATH" +# NB See the docdir handling in install_components for an explanation of this +# weird string +valopt docdir "\" "install documentation in PATH" +opt ldconfig 1 "run ldconfig after installation (Linux only)" +opt verify 1 "obsolete" +flag verbose "run with verbose output" + +if [ $HELP -eq 1 ] +then + echo + exit 0 +fi + +verbose_step_msg "validating arguments" +validate_opt + +# Template configuration. +# These names surrounded by '%%` are replaced by sed when generating install.sh +# FIXME: Might want to consider loading this from a file and not generating install.sh + +# Rust or Cargo +TEMPLATE_PRODUCT_NAME=%%TEMPLATE_PRODUCT_NAME%% +# rustlib or cargo +TEMPLATE_REL_MANIFEST_DIR=%%TEMPLATE_REL_MANIFEST_DIR%% +# 'Rust is ready to roll.' or 'Cargo is cool to cruise.' +TEMPLATE_SUCCESS_MESSAGE=%%TEMPLATE_SUCCESS_MESSAGE%% +# Locations to look for directories containing legacy, pre-versioned manifests +TEMPLATE_LEGACY_MANIFEST_DIRS=%%TEMPLATE_LEGACY_MANIFEST_DIRS%% +# The installer version +TEMPLATE_RUST_INSTALLER_VERSION=%%TEMPLATE_RUST_INSTALLER_VERSION%% + +# OK, let's get installing ... + +# This is where we are installing from +src_dir="$(abs_path $(dirname "$0"))" + +# The name of the script +src_basename="$(basename "$0")" + +# If we've been run as 'uninstall.sh' (from the existing installation) +# then we're doing a full uninstall, as opposed to the --uninstall flag +# which just means 'uninstall my components'. +if [ "$src_basename" = "uninstall.sh" ]; then + if [ "${*:-}" != "" ]; then + # Currently don't know what to do with arguments in this mode + err "uninstall.sh does not take any arguments" + fi + CFG_UNINSTALL=1 + CFG_DESTDIR_PREFIX="$(abs_path "$src_dir/../../")" + CFG_LIBDIR="$(abs_path "$src_dir/../")" +fi + +# This is where we are installing to +dest_prefix="$CFG_DESTDIR_PREFIX" + +# Open the components file to get the list of components to install. +# NB: During install this components file is read from the installer's +# source dir, during a full uninstall it's read from the manifest dir, +# and thus contains all installed components. +components=`cat "$src_dir/components"` + +# Sanity check: do we have components? +if [ ! -n "$components" ]; then + err "unable to find installation components" +fi + +# If the user asked for a component list, do that and exit +if [ -n "${CFG_LIST_COMPONENTS-}" ]; then + echo + echo "# Available components" + echo + for component in $components; do + echo "* $component" + done + echo + exit 0 +fi + +# If the user specified which components to install/uninstall, +# then validate that they exist and select them for installation +if [ -n "$CFG_COMPONENTS" ]; then + # Remove commas + user_components="$(echo "$CFG_COMPONENTS" | sed "s/,/ /g")" + for user_component in $user_components; do + found=false + for my_component in $components; do + if [ "$user_component" = "$my_component" ]; then + found=true + fi + done + if [ "$found" = false ]; then + err "unknown component: $user_component" + fi + done + components="$user_components" +fi + +if [ -n "$CFG_WITHOUT" ]; then + without_components="$(echo "$CFG_WITHOUT" | sed "s/,/ /g")" + + # This does **not** check that all components in without_components are + # actually present in the list of available components. + # + # Currently that's considered good as it makes it easier to be compatible + # with multiple Rust versions (which may change the exact list of + # components) when writing install scripts. + new_comp="" + for component in $components; do + found=false + for my_component in $without_components; do + if [ "$component" = "$my_component" ]; then + found=true + fi + done + if [ "$found" = false ]; then + # If we didn't find the component in without, then add it to new list. + new_comp="$new_comp $component" + fi + done + components="$new_comp" +fi + +if [ -z "$components" ]; then + if [ -z "${CFG_UNINSTALL-}" ]; then + err "no components selected for installation" + else + err "no components selected for uninstallation" + fi +fi + +do_preflight_sanity_checks "$src_dir" "$dest_prefix" + +# Using an absolute path to libdir in a few places so that the status +# messages are consistently using absolute paths. +absolutify "$CFG_LIBDIR" +abs_libdir="$RETVAL" +assert_nz "$abs_libdir" "abs_libdir" + +# Create the manifest directory, where we will put our logs +make_dir_recursive "$abs_libdir/$TEMPLATE_REL_MANIFEST_DIR" +need_ok "failed to create $TEMPLATE_REL_MANIFEST_DIR" + +# Log messages and commands +init_logging "$abs_libdir" + +# First do any uninstallation, including from legacy manifests. This +# will also upgrade the metadata of existing installs. +uninstall_components "$abs_libdir" "$dest_prefix" "$components" + +# If we're only uninstalling then exit +if [ -n "${CFG_UNINSTALL-}" ] +then + echo + echo " $TEMPLATE_PRODUCT_NAME is uninstalled." + echo + exit 0 +fi + +# Create the manifest directory again! uninstall_legacy +# may have deleted it. +make_dir_recursive "$abs_libdir/$TEMPLATE_REL_MANIFEST_DIR" +need_ok "failed to create $TEMPLATE_REL_MANIFEST_DIR" + +# Drop the version number into the manifest dir +write_to_file "$TEMPLATE_RUST_INSTALLER_VERSION" "$abs_libdir/$TEMPLATE_REL_MANIFEST_DIR/rust-installer-version" +critical_need_ok "failed to write installer version" + +# Install the uninstaller +install_uninstaller "$src_dir" "$src_basename" "$abs_libdir" + +# Install each component +install_components "$src_dir" "$abs_libdir" "$dest_prefix" "$components" + +# Make dynamic libraries available to the linker +maybe_configure_ld "$abs_libdir" + +echo +echo " $TEMPLATE_SUCCESS_MESSAGE" +echo + + diff --git a/src/tools/rust-installer/make-tarballs.sh b/src/tools/rust-installer/make-tarballs.sh new file mode 100755 index 0000000000000..e9f88cc8b7186 --- /dev/null +++ b/src/tools/rust-installer/make-tarballs.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright 2014 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ue + +# Prints the absolute path of a directory to stdout +abs_path() { + local path="$1" + # Unset CDPATH because it causes havok: it makes the destination unpredictable + # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null + # for good measure. + (unset CDPATH && cd "$path" > /dev/null && pwd) +} + +src_dir="$(abs_path $(dirname "$0"))" +cargo run --manifest-path="$src_dir/Cargo.toml" -- tarball "$@" diff --git a/src/tools/rust-installer/rust-installer-version b/src/tools/rust-installer/rust-installer-version new file mode 100644 index 0000000000000..e440e5c842586 --- /dev/null +++ b/src/tools/rust-installer/rust-installer-version @@ -0,0 +1 @@ +3 \ No newline at end of file diff --git a/src/tools/rust-installer/src/combiner.rs b/src/tools/rust-installer/src/combiner.rs new file mode 100644 index 0000000000000..2ec09d67e3e62 --- /dev/null +++ b/src/tools/rust-installer/src/combiner.rs @@ -0,0 +1,161 @@ +use super::Scripter; +use super::Tarballer; +use crate::{ + compression::{CompressionFormat, CompressionFormats}, + util::*, +}; +use anyhow::{bail, Context, Result}; +use std::io::{Read, Write}; +use std::path::Path; +use tar::Archive; + +actor! { + #[derive(Debug)] + pub struct Combiner { + /// The name of the product, for display. + #[clap(value_name = "NAME")] + product_name: String = "Product", + + /// The name of the package tarball. + #[clap(value_name = "NAME")] + package_name: String = "package", + + /// The directory under lib/ where the manifest lives. + #[clap(value_name = "DIR")] + rel_manifest_dir: String = "packagelib", + + /// The string to print after successful installation. + #[clap(value_name = "MESSAGE")] + success_message: String = "Installed.", + + /// Places to look for legacy manifests to uninstall. + #[clap(value_name = "DIRS")] + legacy_manifest_dirs: String = "", + + /// Installers to combine. + #[clap(value_name = "FILE,FILE")] + input_tarballs: String = "", + + /// Directory containing files that should not be installed. + #[clap(value_name = "DIR")] + non_installed_overlay: String = "", + + /// The directory to do temporary work. + #[clap(value_name = "DIR")] + work_dir: String = "./workdir", + + /// The location to put the final image and tarball. + #[clap(value_name = "DIR")] + output_dir: String = "./dist", + + /// The formats used to compress the tarball + #[clap(value_name = "FORMAT", default_value_t)] + compression_formats: CompressionFormats, + } +} + +impl Combiner { + /// Combines the installer tarballs. + pub fn run(self) -> Result<()> { + create_dir_all(&self.work_dir)?; + + let package_dir = Path::new(&self.work_dir).join(&self.package_name); + if package_dir.exists() { + remove_dir_all(&package_dir)?; + } + create_dir_all(&package_dir)?; + + // Merge each installer into the work directory of the new installer. + let components = create_new_file(package_dir.join("components"))?; + for input_tarball in self + .input_tarballs + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + { + // Extract the input tarballs + let compression = + CompressionFormat::detect_from_path(input_tarball).ok_or_else(|| { + anyhow::anyhow!("couldn't figure out the format of {}", input_tarball) + })?; + Archive::new(compression.decode(input_tarball)?) + .unpack(&self.work_dir) + .with_context(|| { + format!( + "unable to extract '{}' into '{}'", + &input_tarball, self.work_dir + ) + })?; + + let pkg_name = + input_tarball.trim_end_matches(&format!(".tar.{}", compression.extension())); + let pkg_name = Path::new(pkg_name).file_name().unwrap(); + let pkg_dir = Path::new(&self.work_dir).join(&pkg_name); + + // Verify the version number. + let mut version = String::new(); + open_file(pkg_dir.join("rust-installer-version")) + .and_then(|mut file| Ok(file.read_to_string(&mut version)?)) + .with_context(|| format!("failed to read version in '{}'", input_tarball))?; + if version.trim().parse() != Ok(crate::RUST_INSTALLER_VERSION) { + bail!("incorrect installer version in {}", input_tarball); + } + + // Copy components to the new combined installer. + let mut pkg_components = String::new(); + open_file(pkg_dir.join("components")) + .and_then(|mut file| Ok(file.read_to_string(&mut pkg_components)?)) + .with_context(|| format!("failed to read components in '{}'", input_tarball))?; + for component in pkg_components.split_whitespace() { + // All we need to do is copy the component directory. We could + // move it, but rustbuild wants to reuse the unpacked package + // dir for OS-specific installers on macOS and Windows. + let component_dir = package_dir.join(&component); + create_dir(&component_dir)?; + copy_recursive(&pkg_dir.join(&component), &component_dir)?; + + // Merge the component name. + writeln!(&components, "{}", component).context("failed to write new components")?; + } + } + drop(components); + + // Write the installer version. + let version = package_dir.join("rust-installer-version"); + writeln!( + create_new_file(version)?, + "{}", + crate::RUST_INSTALLER_VERSION + ) + .context("failed to write new installer version")?; + + // Copy the overlay. + if !self.non_installed_overlay.is_empty() { + copy_recursive(self.non_installed_overlay.as_ref(), &package_dir)?; + } + + // Generate the install script. + let output_script = package_dir.join("install.sh"); + let mut scripter = Scripter::default(); + scripter + .product_name(self.product_name) + .rel_manifest_dir(self.rel_manifest_dir) + .success_message(self.success_message) + .legacy_manifest_dirs(self.legacy_manifest_dirs) + .output_script(path_to_str(&output_script)?.into()); + scripter.run()?; + + // Make the tarballs. + create_dir_all(&self.output_dir)?; + let output = Path::new(&self.output_dir).join(&self.package_name); + let mut tarballer = Tarballer::default(); + tarballer + .work_dir(self.work_dir) + .input(self.package_name) + .output(path_to_str(&output)?.into()) + .compression_formats(self.compression_formats.clone()); + tarballer.run()?; + + Ok(()) + } +} diff --git a/src/tools/rust-installer/src/compression.rs b/src/tools/rust-installer/src/compression.rs new file mode 100644 index 0000000000000..9b176982d000d --- /dev/null +++ b/src/tools/rust-installer/src/compression.rs @@ -0,0 +1,203 @@ +use anyhow::{Context, Error}; +use flate2::{read::GzDecoder, write::GzEncoder}; +use rayon::prelude::*; +use std::{convert::TryFrom, fmt, io::Read, io::Write, path::Path, str::FromStr}; +use xz2::{read::XzDecoder, write::XzEncoder}; + +#[derive(Debug, Copy, Clone)] +pub enum CompressionFormat { + Gz, + Xz, +} + +impl CompressionFormat { + pub(crate) fn detect_from_path(path: impl AsRef) -> Option { + match path.as_ref().extension().and_then(|e| e.to_str()) { + Some("gz") => Some(CompressionFormat::Gz), + Some("xz") => Some(CompressionFormat::Xz), + _ => None, + } + } + + pub(crate) fn extension(&self) -> &'static str { + match self { + CompressionFormat::Gz => "gz", + CompressionFormat::Xz => "xz", + } + } + + pub(crate) fn encode(&self, path: impl AsRef) -> Result, Error> { + let mut os = path.as_ref().as_os_str().to_os_string(); + os.push(format!(".{}", self.extension())); + let path = Path::new(&os); + + if path.exists() { + crate::util::remove_file(path)?; + } + let file = crate::util::create_new_file(path)?; + + Ok(match self { + CompressionFormat::Gz => Box::new(GzEncoder::new(file, flate2::Compression::best())), + CompressionFormat::Xz => { + let mut filters = xz2::stream::Filters::new(); + // the preset is overridden by the other options so it doesn't matter + let mut lzma_ops = xz2::stream::LzmaOptions::new_preset(9).unwrap(); + // This sets the overall dictionary size, which is also how much memory (baseline) + // is needed for decompression. + lzma_ops.dict_size(64 * 1024 * 1024); + // Use the best match finder for compression ratio. + lzma_ops.match_finder(xz2::stream::MatchFinder::BinaryTree4); + lzma_ops.mode(xz2::stream::Mode::Normal); + // Set nice len to the maximum for best compression ratio + lzma_ops.nice_len(273); + // Set depth to a reasonable value, 0 means auto, 1000 is somwhat high but gives + // good results. + lzma_ops.depth(1000); + // 2 is the default and does well for most files + lzma_ops.position_bits(2); + // 0 is the default and does well for most files + lzma_ops.literal_position_bits(0); + // 3 is the default and does well for most files + lzma_ops.literal_context_bits(3); + + filters.lzma2(&lzma_ops); + let compressor = XzEncoder::new_stream( + std::io::BufWriter::new(file), + xz2::stream::MtStreamBuilder::new() + .threads(1) + .filters(filters) + .encoder() + .unwrap(), + ); + Box::new(compressor) + } + }) + } + + pub(crate) fn decode(&self, path: impl AsRef) -> Result, Error> { + let file = crate::util::open_file(path.as_ref())?; + Ok(match self { + CompressionFormat::Gz => Box::new(GzDecoder::new(file)), + CompressionFormat::Xz => Box::new(XzDecoder::new(file)), + }) + } +} + +/// This struct wraps Vec in order to parse the value from the command line. +#[derive(Debug, Clone)] +pub struct CompressionFormats(Vec); + +impl TryFrom<&'_ str> for CompressionFormats { + type Error = Error; + + fn try_from(value: &str) -> Result { + let mut parsed = Vec::new(); + for format in value.split(',') { + match format.trim() { + "gz" => parsed.push(CompressionFormat::Gz), + "xz" => parsed.push(CompressionFormat::Xz), + other => anyhow::bail!("unknown compression format: {}", other), + } + } + Ok(CompressionFormats(parsed)) + } +} + +impl FromStr for CompressionFormats { + type Err = Error; + + fn from_str(value: &str) -> Result { + Self::try_from(value) + } +} + +impl fmt::Display for CompressionFormats { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (i, format) in self.iter().enumerate() { + if i != 0 { + write!(f, ",")?; + } + fmt::Display::fmt( + match format { + CompressionFormat::Xz => "xz", + CompressionFormat::Gz => "gz", + }, + f, + )?; + } + Ok(()) + } +} + +impl Default for CompressionFormats { + fn default() -> Self { + Self(vec![CompressionFormat::Gz, CompressionFormat::Xz]) + } +} + +impl CompressionFormats { + pub(crate) fn iter(&self) -> impl Iterator + '_ { + self.0.iter().map(|i| *i) + } +} + +pub(crate) trait Encoder: Send + Write { + fn finish(self: Box) -> Result<(), Error>; +} + +impl Encoder for GzEncoder { + fn finish(self: Box) -> Result<(), Error> { + GzEncoder::finish(*self).context("failed to finish .gz file")?; + Ok(()) + } +} + +impl Encoder for XzEncoder { + fn finish(self: Box) -> Result<(), Error> { + XzEncoder::finish(*self).context("failed to finish .xz file")?; + Ok(()) + } +} + +pub(crate) struct CombinedEncoder { + encoders: Vec>, +} + +impl CombinedEncoder { + pub(crate) fn new(encoders: Vec>) -> Box { + Box::new(Self { encoders }) + } +} + +impl Write for CombinedEncoder { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> { + self.encoders + .par_iter_mut() + .map(|w| w.write_all(buf)) + .collect::>>()?; + Ok(()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.encoders + .par_iter_mut() + .map(|w| w.flush()) + .collect::>>()?; + Ok(()) + } +} + +impl Encoder for CombinedEncoder { + fn finish(self: Box) -> Result<(), Error> { + self.encoders + .into_par_iter() + .map(|e| e.finish()) + .collect::, Error>>()?; + Ok(()) + } +} diff --git a/src/tools/rust-installer/src/generator.rs b/src/tools/rust-installer/src/generator.rs new file mode 100644 index 0000000000000..1e4d00b0553aa --- /dev/null +++ b/src/tools/rust-installer/src/generator.rs @@ -0,0 +1,178 @@ +use super::Scripter; +use super::Tarballer; +use crate::compression::CompressionFormats; +use crate::util::*; +use anyhow::{bail, format_err, Context, Result}; +use std::collections::BTreeSet; +use std::io::Write; +use std::path::Path; + +actor! { + #[derive(Debug)] + pub struct Generator { + /// The name of the product, for display + #[clap(value_name = "NAME")] + product_name: String = "Product", + + /// The name of the component, distinct from other installed components + #[clap(value_name = "NAME")] + component_name: String = "component", + + /// The name of the package, tarball + #[clap(value_name = "NAME")] + package_name: String = "package", + + /// The directory under lib/ where the manifest lives + #[clap(value_name = "DIR")] + rel_manifest_dir: String = "packagelib", + + /// The string to print after successful installation + #[clap(value_name = "MESSAGE")] + success_message: String = "Installed.", + + /// Places to look for legacy manifests to uninstall + #[clap(value_name = "DIRS")] + legacy_manifest_dirs: String = "", + + /// Directory containing files that should not be installed + #[clap(value_name = "DIR")] + non_installed_overlay: String = "", + + /// Path prefixes of directories that should be installed/uninstalled in bulk + #[clap(value_name = "DIRS")] + bulk_dirs: String = "", + + /// The directory containing the installation medium + #[clap(value_name = "DIR")] + image_dir: String = "./install_image", + + /// The directory to do temporary work + #[clap(value_name = "DIR")] + work_dir: String = "./workdir", + + /// The location to put the final image and tarball + #[clap(value_name = "DIR")] + output_dir: String = "./dist", + + /// The formats used to compress the tarball + #[clap(value_name = "FORMAT", default_value_t)] + compression_formats: CompressionFormats, + } +} + +impl Generator { + /// Generates the actual installer tarball + pub fn run(self) -> Result<()> { + create_dir_all(&self.work_dir)?; + + let package_dir = Path::new(&self.work_dir).join(&self.package_name); + if package_dir.exists() { + remove_dir_all(&package_dir)?; + } + + // Copy the image and write the manifest + let component_dir = package_dir.join(&self.component_name); + create_dir_all(&component_dir)?; + copy_and_manifest(self.image_dir.as_ref(), &component_dir, &self.bulk_dirs)?; + + // Write the component name + let components = package_dir.join("components"); + writeln!(create_new_file(components)?, "{}", self.component_name) + .context("failed to write the component file")?; + + // Write the installer version (only used by combine-installers.sh) + let version = package_dir.join("rust-installer-version"); + writeln!( + create_new_file(version)?, + "{}", + crate::RUST_INSTALLER_VERSION + ) + .context("failed to write new installer version")?; + + // Copy the overlay + if !self.non_installed_overlay.is_empty() { + copy_recursive(self.non_installed_overlay.as_ref(), &package_dir)?; + } + + // Generate the install script + let output_script = package_dir.join("install.sh"); + let mut scripter = Scripter::default(); + scripter + .product_name(self.product_name) + .rel_manifest_dir(self.rel_manifest_dir) + .success_message(self.success_message) + .legacy_manifest_dirs(self.legacy_manifest_dirs) + .output_script(path_to_str(&output_script)?.into()); + scripter.run()?; + + // Make the tarballs + create_dir_all(&self.output_dir)?; + let output = Path::new(&self.output_dir).join(&self.package_name); + let mut tarballer = Tarballer::default(); + tarballer + .work_dir(self.work_dir) + .input(self.package_name) + .output(path_to_str(&output)?.into()) + .compression_formats(self.compression_formats.clone()); + tarballer.run()?; + + Ok(()) + } +} + +/// Copies the `src` directory recursively to `dst`, writing `manifest.in` too. +fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> Result<()> { + let mut manifest = create_new_file(dst.join("manifest.in"))?; + let bulk_dirs: Vec<_> = bulk_dirs + .split(',') + .filter(|s| !s.is_empty()) + .map(Path::new) + .collect(); + + let mut paths = BTreeSet::new(); + copy_with_callback(src, dst, |path, file_type| { + // We need paths to be compatible with both Unix and Windows. + if path + .components() + .filter_map(|c| c.as_os_str().to_str()) + .any(|s| s.contains('\\')) + { + bail!( + "rust-installer doesn't support '\\' in path components: {:?}", + path + ); + } + + // Normalize to Unix-style path separators. + let normalized_string; + let mut string = path.to_str().ok_or_else(|| { + format_err!( + "rust-installer doesn't support non-Unicode paths: {:?}", + path + ) + })?; + if string.contains('\\') { + normalized_string = string.replace('\\', "/"); + string = &normalized_string; + } + + if file_type.is_dir() { + // Only manifest directories that are explicitly bulk. + if bulk_dirs.contains(&path) { + paths.insert(format!("dir:{}\n", string)); + } + } else { + // Only manifest files that aren't under bulk directories. + if !bulk_dirs.iter().any(|d| path.starts_with(d)) { + paths.insert(format!("file:{}\n", string)); + } + } + Ok(()) + })?; + + for path in paths { + manifest.write_all(path.as_bytes())?; + } + + Ok(()) +} diff --git a/src/tools/rust-installer/src/lib.rs b/src/tools/rust-installer/src/lib.rs new file mode 100644 index 0000000000000..7990920192a6c --- /dev/null +++ b/src/tools/rust-installer/src/lib.rs @@ -0,0 +1,17 @@ +#[macro_use] +mod util; + +mod combiner; +mod compression; +mod generator; +mod scripter; +mod tarballer; + +pub use crate::combiner::Combiner; +pub use crate::generator::Generator; +pub use crate::scripter::Scripter; +pub use crate::tarballer::Tarballer; + +/// The installer version, output only to be used by combine-installers.sh. +/// (should match `SOURCE_DIRECTORY/rust_installer_version`) +pub const RUST_INSTALLER_VERSION: u32 = 3; diff --git a/src/tools/rust-installer/src/main.rs b/src/tools/rust-installer/src/main.rs new file mode 100644 index 0000000000000..be8a0d68343e4 --- /dev/null +++ b/src/tools/rust-installer/src/main.rs @@ -0,0 +1,27 @@ +use anyhow::{Context, Result}; +use clap::{self, Parser}; + +#[derive(Parser)] +struct CommandLine { + #[clap(subcommand)] + command: Subcommand, +} + +#[derive(clap::Subcommand)] +enum Subcommand { + Generate(installer::Generator), + Combine(installer::Combiner), + Script(installer::Scripter), + Tarball(installer::Tarballer), +} + +fn main() -> Result<()> { + let command_line = CommandLine::parse(); + match command_line.command { + Subcommand::Combine(combiner) => combiner.run().context("failed to combine installers")?, + Subcommand::Generate(generator) => generator.run().context("failed to generate installer")?, + Subcommand::Script(scripter) => scripter.run().context("failed to generate installation script")?, + Subcommand::Tarball(tarballer) => tarballer.run().context("failed to generate tarballs")?, + } + Ok(()) +} diff --git a/src/tools/rust-installer/src/remove_dir_all.rs b/src/tools/rust-installer/src/remove_dir_all.rs new file mode 100644 index 0000000000000..11097652865c4 --- /dev/null +++ b/src/tools/rust-installer/src/remove_dir_all.rs @@ -0,0 +1,860 @@ +#![allow(non_snake_case)] + +use std::io; +use std::path::Path; + +#[cfg(not(windows))] +pub fn remove_dir_all(path: &Path) -> io::Result<()> { + ::std::fs::remove_dir_all(path) +} + +#[cfg(windows)] +pub fn remove_dir_all(path: &Path) -> io::Result<()> { + win::remove_dir_all(path) +} + +#[cfg(windows)] +mod win { + use winapi::ctypes::{c_uint, c_ushort}; + use winapi::shared::minwindef::{BOOL, DWORD, FALSE, FILETIME, LPVOID}; + use winapi::shared::winerror::{ + ERROR_CALL_NOT_IMPLEMENTED, ERROR_INSUFFICIENT_BUFFER, ERROR_NO_MORE_FILES, + }; + use winapi::um::errhandlingapi::{GetLastError, SetLastError}; + use winapi::um::fileapi::{ + CreateFileW, FindFirstFileW, FindNextFileW, GetFileInformationByHandle, + }; + use winapi::um::fileapi::{BY_HANDLE_FILE_INFORMATION, CREATE_ALWAYS, CREATE_NEW}; + use winapi::um::fileapi::{FILE_BASIC_INFO, FILE_RENAME_INFO, TRUNCATE_EXISTING}; + use winapi::um::fileapi::{OPEN_ALWAYS, OPEN_EXISTING}; + use winapi::um::handleapi::{CloseHandle, INVALID_HANDLE_VALUE}; + use winapi::um::ioapiset::DeviceIoControl; + use winapi::um::libloaderapi::{GetModuleHandleW, GetProcAddress}; + use winapi::um::minwinbase::{ + FileBasicInfo, FileRenameInfo, FILE_INFO_BY_HANDLE_CLASS, WIN32_FIND_DATAW, + }; + use winapi::um::winbase::SECURITY_SQOS_PRESENT; + use winapi::um::winbase::{ + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_DELETE_ON_CLOSE, FILE_FLAG_OPEN_REPARSE_POINT, + }; + use winapi::um::winioctl::FSCTL_GET_REPARSE_POINT; + use winapi::um::winnt::{DELETE, FILE_ATTRIBUTE_DIRECTORY, HANDLE, LPCWSTR}; + use winapi::um::winnt::{FILE_ATTRIBUTE_READONLY, FILE_ATTRIBUTE_REPARSE_POINT}; + use winapi::um::winnt::{FILE_GENERIC_WRITE, FILE_WRITE_DATA, GENERIC_READ, GENERIC_WRITE}; + use winapi::um::winnt::{FILE_READ_ATTRIBUTES, FILE_WRITE_ATTRIBUTES}; + use winapi::um::winnt::{FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE}; + use winapi::um::winnt::{IO_REPARSE_TAG_MOUNT_POINT, IO_REPARSE_TAG_SYMLINK, LARGE_INTEGER}; + + use std::ffi::{OsStr, OsString}; + use std::io; + use std::mem; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; + use std::path::{Path, PathBuf}; + use std::ptr; + use std::sync::Arc; + + pub fn remove_dir_all(path: &Path) -> io::Result<()> { + // On Windows it is not enough to just recursively remove the contents of a + // directory and then the directory itself. Deleting does not happen + // instantaneously, but is scheduled. + // To work around this, we move the file or directory to some `base_dir` + // right before deletion to avoid races. + // + // As `base_dir` we choose the parent dir of the directory we want to + // remove. We very probably have permission to create files here, as we + // already need write permission in this dir to delete the directory. And it + // should be on the same volume. + // + // To handle files with names like `CON` and `morse .. .`, and when a + // directory structure is so deep it needs long path names the path is first + // converted to a `//?/`-path with `get_path()`. + // + // To make sure we don't leave a moved file laying around if the process + // crashes before we can delete the file, we do all operations on an file + // handle. By opening a file with `FILE_FLAG_DELETE_ON_CLOSE` Windows will + // always delete the file when the handle closes. + // + // All files are renamed to be in the `base_dir`, and have their name + // changed to "rm-". After every rename the counter is increased. + // Rename should not overwrite possibly existing files in the base dir. So + // if it fails with `AlreadyExists`, we just increase the counter and try + // again. + // + // For read-only files and directories we first have to remove the read-only + // attribute before we can move or delete them. This also removes the + // attribute from possible hardlinks to the file, so just before closing we + // restore the read-only attribute. + // + // If 'path' points to a directory symlink or junction we should not + // recursively remove the target of the link, but only the link itself. + // + // Moving and deleting is guaranteed to succeed if we are able to open the + // file with `DELETE` permission. If others have the file open we only have + // `DELETE` permission if they have specified `FILE_SHARE_DELETE`. We can + // also delete the file now, but it will not disappear until all others have + // closed the file. But no-one can open the file after we have flagged it + // for deletion. + + // Open the path once to get the canonical path, file type and attributes. + let (path, metadata) = { + let mut opts = OpenOptions::new(); + opts.access_mode(FILE_READ_ATTRIBUTES); + opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT); + let file = File::open(path, &opts)?; + (get_path(&file)?, file.file_attr()?) + }; + + let mut ctx = RmdirContext { + base_dir: match path.parent() { + Some(dir) => dir, + None => { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "can't delete root directory", + )) + } + }, + readonly: metadata.perm().readonly(), + counter: 0, + }; + + let filetype = metadata.file_type(); + if filetype.is_dir() { + remove_dir_all_recursive(path.as_ref(), &mut ctx) + } else if filetype.is_symlink_dir() { + remove_item(path.as_ref(), &mut ctx) + } else { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "Not a directory", + )) + } + } + + fn readdir(p: &Path) -> io::Result { + let root = p.to_path_buf(); + let star = p.join("*"); + let path = to_u16s(&star)?; + + unsafe { + let mut wfd = mem::zeroed(); + let find_handle = FindFirstFileW(path.as_ptr(), &mut wfd); + if find_handle != INVALID_HANDLE_VALUE { + Ok(ReadDir { + handle: FindNextFileHandle(find_handle), + root: Arc::new(root), + first: Some(wfd), + }) + } else { + Err(io::Error::last_os_error()) + } + } + } + + struct RmdirContext<'a> { + base_dir: &'a Path, + readonly: bool, + counter: u64, + } + + fn remove_dir_all_recursive(path: &Path, ctx: &mut RmdirContext) -> io::Result<()> { + let dir_readonly = ctx.readonly; + for child in readdir(path)? { + let child = child?; + let child_type = child.file_type()?; + ctx.readonly = child.metadata()?.perm().readonly(); + if child_type.is_dir() { + remove_dir_all_recursive(&child.path(), ctx)?; + } else { + remove_item(&child.path().as_ref(), ctx)?; + } + } + ctx.readonly = dir_readonly; + remove_item(path, ctx) + } + + fn remove_item(path: &Path, ctx: &mut RmdirContext) -> io::Result<()> { + if !ctx.readonly { + let mut opts = OpenOptions::new(); + opts.access_mode(DELETE); + opts.custom_flags( + FILE_FLAG_BACKUP_SEMANTICS | // delete directory + FILE_FLAG_OPEN_REPARSE_POINT | // delete symlink + FILE_FLAG_DELETE_ON_CLOSE, + ); + let file = File::open(path, &opts)?; + move_item(&file, ctx) + } else { + // remove read-only permision + set_perm(&path, FilePermissions::new())?; + // move and delete file, similar to !readonly. + // only the access mode is different. + let mut opts = OpenOptions::new(); + opts.access_mode(DELETE | FILE_WRITE_ATTRIBUTES); + opts.custom_flags( + FILE_FLAG_BACKUP_SEMANTICS + | FILE_FLAG_OPEN_REPARSE_POINT + | FILE_FLAG_DELETE_ON_CLOSE, + ); + let file = File::open(path, &opts)?; + move_item(&file, ctx)?; + // restore read-only flag just in case there are other hard links + let mut perm = FilePermissions::new(); + perm.set_readonly(true); + let _ = file.set_perm(perm); // ignore if this fails + Ok(()) + } + } + + macro_rules! compat_fn { + ($module:ident: $( + fn $symbol:ident($($argname:ident: $argtype:ty),*) + -> $rettype:ty { + $($body:expr);* + } + )*) => ($( + #[allow(unused_variables)] + unsafe fn $symbol($($argname: $argtype),*) -> $rettype { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::mem; + use std::ffi::CString; + type F = unsafe extern "system" fn($($argtype),*) -> $rettype; + + lazy_static! { static ref PTR: AtomicUsize = AtomicUsize::new(0);} + + fn lookup(module: &str, symbol: &str) -> Option { + let mut module: Vec = module.encode_utf16().collect(); + module.push(0); + let symbol = CString::new(symbol).unwrap(); + unsafe { + let handle = GetModuleHandleW(module.as_ptr()); + match GetProcAddress(handle, symbol.as_ptr()) as usize { + 0 => None, + n => Some(n), + } + } + } + + fn store_func(ptr: &AtomicUsize, module: &str, symbol: &str, + fallback: usize) -> usize { + let value = lookup(module, symbol).unwrap_or(fallback); + ptr.store(value, Ordering::SeqCst); + value + } + + fn load() -> usize { + store_func(&PTR, stringify!($module), stringify!($symbol), fallback as usize) + } + unsafe extern "system" fn fallback($($argname: $argtype),*) + -> $rettype { + $($body);* + } + + let addr = match PTR.load(Ordering::SeqCst) { + 0 => load(), + n => n, + }; + mem::transmute::(addr)($($argname),*) + } + )*) + } + + compat_fn! { + kernel32: + fn GetFinalPathNameByHandleW(_hFile: HANDLE, + _lpszFilePath: LPCWSTR, + _cchFilePath: DWORD, + _dwFlags: DWORD) -> DWORD { + SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 + } + fn SetFileInformationByHandle(_hFile: HANDLE, + _FileInformationClass: FILE_INFO_BY_HANDLE_CLASS, + _lpFileInformation: LPVOID, + _dwBufferSize: DWORD) -> BOOL { + SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 + } + } + + fn cvt(i: i32) -> io::Result { + if i == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(i) + } + } + + fn to_u16s>(s: S) -> io::Result> { + fn inner(s: &OsStr) -> io::Result> { + let mut maybe_result: Vec = s.encode_wide().collect(); + if maybe_result.iter().any(|&u| u == 0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "strings passed to WinAPI cannot contain NULs", + )); + } + maybe_result.push(0); + Ok(maybe_result) + } + inner(s.as_ref()) + } + + fn truncate_utf16_at_nul<'a>(v: &'a [u16]) -> &'a [u16] { + match v.iter().position(|c| *c == 0) { + // don't include the 0 + Some(i) => &v[..i], + None => v, + } + } + + fn fill_utf16_buf(mut f1: F1, f2: F2) -> io::Result + where + F1: FnMut(*mut u16, DWORD) -> DWORD, + F2: FnOnce(&[u16]) -> T, + { + // Start off with a stack buf but then spill over to the heap if we end up + // needing more space. + let mut stack_buf = [0u16; 512]; + let mut heap_buf = Vec::new(); + unsafe { + let mut n = stack_buf.len(); + loop { + let buf = if n <= stack_buf.len() { + &mut stack_buf[..] + } else { + let extra = n - heap_buf.len(); + heap_buf.reserve(extra); + heap_buf.set_len(n); + &mut heap_buf[..] + }; + + // This function is typically called on windows API functions which + // will return the correct length of the string, but these functions + // also return the `0` on error. In some cases, however, the + // returned "correct length" may actually be 0! + // + // To handle this case we call `SetLastError` to reset it to 0 and + // then check it again if we get the "0 error value". If the "last + // error" is still 0 then we interpret it as a 0 length buffer and + // not an actual error. + SetLastError(0); + let k = match f1(buf.as_mut_ptr(), n as DWORD) { + 0 if GetLastError() == 0 => 0, + 0 => return Err(io::Error::last_os_error()), + n => n, + } as usize; + if k == n && GetLastError() == ERROR_INSUFFICIENT_BUFFER { + n *= 2; + } else if k >= n { + n = k; + } else { + return Ok(f2(&buf[..k])); + } + } + } + } + + #[derive(Clone, PartialEq, Eq, Debug, Default)] + struct FilePermissions { + readonly: bool, + } + + impl FilePermissions { + fn new() -> FilePermissions { + Default::default() + } + fn readonly(&self) -> bool { + self.readonly + } + fn set_readonly(&mut self, readonly: bool) { + self.readonly = readonly + } + } + + #[derive(Clone)] + struct OpenOptions { + // generic + read: bool, + write: bool, + append: bool, + truncate: bool, + create: bool, + create_new: bool, + // system-specific + custom_flags: u32, + access_mode: Option, + attributes: DWORD, + share_mode: DWORD, + security_qos_flags: DWORD, + security_attributes: usize, // FIXME: should be a reference + } + + impl OpenOptions { + fn new() -> OpenOptions { + OpenOptions { + // generic + read: false, + write: false, + append: false, + truncate: false, + create: false, + create_new: false, + // system-specific + custom_flags: 0, + access_mode: None, + share_mode: FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + attributes: 0, + security_qos_flags: 0, + security_attributes: 0, + } + } + fn custom_flags(&mut self, flags: u32) { + self.custom_flags = flags; + } + fn access_mode(&mut self, access_mode: u32) { + self.access_mode = Some(access_mode); + } + + fn get_access_mode(&self) -> io::Result { + const ERROR_INVALID_PARAMETER: i32 = 87; + + match (self.read, self.write, self.append, self.access_mode) { + (_, _, _, Some(mode)) => Ok(mode), + (true, false, false, None) => Ok(GENERIC_READ), + (false, true, false, None) => Ok(GENERIC_WRITE), + (true, true, false, None) => Ok(GENERIC_READ | GENERIC_WRITE), + (false, _, true, None) => Ok(FILE_GENERIC_WRITE & !FILE_WRITE_DATA), + (true, _, true, None) => Ok(GENERIC_READ | (FILE_GENERIC_WRITE & !FILE_WRITE_DATA)), + (false, false, false, None) => { + Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER)) + } + } + } + + fn get_creation_mode(&self) -> io::Result { + const ERROR_INVALID_PARAMETER: i32 = 87; + + match (self.write, self.append) { + (true, false) => {} + (false, false) => { + if self.truncate || self.create || self.create_new { + return Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER)); + } + } + (_, true) => { + if self.truncate && !self.create_new { + return Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER)); + } + } + } + + Ok(match (self.create, self.truncate, self.create_new) { + (false, false, false) => OPEN_EXISTING, + (true, false, false) => OPEN_ALWAYS, + (false, true, false) => TRUNCATE_EXISTING, + (true, true, false) => CREATE_ALWAYS, + (_, _, true) => CREATE_NEW, + }) + } + + fn get_flags_and_attributes(&self) -> DWORD { + self.custom_flags + | self.attributes + | self.security_qos_flags + | if self.security_qos_flags != 0 { + SECURITY_SQOS_PRESENT + } else { + 0 + } + | if self.create_new { + FILE_FLAG_OPEN_REPARSE_POINT + } else { + 0 + } + } + } + + struct File { + handle: Handle, + } + + impl File { + fn open(path: &Path, opts: &OpenOptions) -> io::Result { + let path = to_u16s(path)?; + let handle = unsafe { + CreateFileW( + path.as_ptr(), + opts.get_access_mode()?, + opts.share_mode, + opts.security_attributes as *mut _, + opts.get_creation_mode()?, + opts.get_flags_and_attributes(), + ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + Err(io::Error::last_os_error()) + } else { + Ok(File { + handle: Handle::new(handle), + }) + } + } + + fn file_attr(&self) -> io::Result { + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = mem::zeroed(); + cvt(GetFileInformationByHandle(self.handle.raw(), &mut info))?; + let mut attr = FileAttr { + attributes: info.dwFileAttributes, + creation_time: info.ftCreationTime, + last_access_time: info.ftLastAccessTime, + last_write_time: info.ftLastWriteTime, + file_size: ((info.nFileSizeHigh as u64) << 32) | (info.nFileSizeLow as u64), + reparse_tag: 0, + }; + if attr.is_reparse_point() { + let mut b = [0; MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; + if let Ok((_, buf)) = self.reparse_point(&mut b) { + attr.reparse_tag = buf.ReparseTag; + } + } + Ok(attr) + } + } + + fn set_attributes(&self, attr: DWORD) -> io::Result<()> { + let zero: LARGE_INTEGER = unsafe { mem::zeroed() }; + + let mut info = FILE_BASIC_INFO { + CreationTime: zero, // do not change + LastAccessTime: zero, // do not change + LastWriteTime: zero, // do not change + ChangeTime: zero, // do not change + FileAttributes: attr, + }; + let size = mem::size_of_val(&info); + cvt(unsafe { + SetFileInformationByHandle( + self.handle.raw(), + FileBasicInfo, + &mut info as *mut _ as *mut _, + size as DWORD, + ) + })?; + Ok(()) + } + + fn rename(&self, new: &Path, replace: bool) -> io::Result<()> { + // &self must be opened with DELETE permission + use std::iter; + #[cfg(target_arch = "x86")] + const STRUCT_SIZE: usize = 12; + #[cfg(target_arch = "x86_64")] + const STRUCT_SIZE: usize = 20; + + // FIXME: check for internal NULs in 'new' + let mut data: Vec = iter::repeat(0u16) + .take(STRUCT_SIZE / 2) + .chain(new.as_os_str().encode_wide()) + .collect(); + data.push(0); + let size = data.len() * 2; + + unsafe { + // Thanks to alignment guarantees on Windows this works + // (8 for 32-bit and 16 for 64-bit) + let info = data.as_mut_ptr() as *mut FILE_RENAME_INFO; + // The type of ReplaceIfExists is BOOL, but it actually expects a + // BOOLEAN. This means true is -1, not c::TRUE. + (*info).ReplaceIfExists = if replace { -1 } else { FALSE }; + (*info).RootDirectory = ptr::null_mut(); + (*info).FileNameLength = (size - STRUCT_SIZE) as DWORD; + cvt(SetFileInformationByHandle( + self.handle().raw(), + FileRenameInfo, + data.as_mut_ptr() as *mut _ as *mut _, + size as DWORD, + ))?; + Ok(()) + } + } + fn set_perm(&self, perm: FilePermissions) -> io::Result<()> { + let attr = self.file_attr()?.attributes; + if perm.readonly == (attr & FILE_ATTRIBUTE_READONLY != 0) { + Ok(()) + } else if perm.readonly { + self.set_attributes(attr | FILE_ATTRIBUTE_READONLY) + } else { + self.set_attributes(attr & !FILE_ATTRIBUTE_READONLY) + } + } + + fn handle(&self) -> &Handle { + &self.handle + } + + fn reparse_point<'a>( + &self, + space: &'a mut [u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE], + ) -> io::Result<(DWORD, &'a REPARSE_DATA_BUFFER)> { + unsafe { + let mut bytes = 0; + cvt({ + DeviceIoControl( + self.handle.raw(), + FSCTL_GET_REPARSE_POINT, + ptr::null_mut(), + 0, + space.as_mut_ptr() as *mut _, + space.len() as DWORD, + &mut bytes, + ptr::null_mut(), + ) + })?; + Ok((bytes, &*(space.as_ptr() as *const REPARSE_DATA_BUFFER))) + } + } + } + + #[derive(Copy, Clone, PartialEq, Eq, Hash)] + enum FileType { + Dir, + File, + SymlinkFile, + SymlinkDir, + ReparsePoint, + MountPoint, + } + + impl FileType { + fn new(attrs: DWORD, reparse_tag: DWORD) -> FileType { + match ( + attrs & FILE_ATTRIBUTE_DIRECTORY != 0, + attrs & FILE_ATTRIBUTE_REPARSE_POINT != 0, + reparse_tag, + ) { + (false, false, _) => FileType::File, + (true, false, _) => FileType::Dir, + (false, true, IO_REPARSE_TAG_SYMLINK) => FileType::SymlinkFile, + (true, true, IO_REPARSE_TAG_SYMLINK) => FileType::SymlinkDir, + (true, true, IO_REPARSE_TAG_MOUNT_POINT) => FileType::MountPoint, + (_, true, _) => FileType::ReparsePoint, + // Note: if a _file_ has a reparse tag of the type IO_REPARSE_TAG_MOUNT_POINT it is + // invalid, as junctions always have to be dirs. We set the filetype to ReparsePoint + // to indicate it is something symlink-like, but not something you can follow. + } + } + + fn is_dir(&self) -> bool { + *self == FileType::Dir + } + fn is_symlink_dir(&self) -> bool { + *self == FileType::SymlinkDir || *self == FileType::MountPoint + } + } + + impl DirEntry { + fn new(root: &Arc, wfd: &WIN32_FIND_DATAW) -> Option { + let first_bytes = &wfd.cFileName[0..3]; + if first_bytes.starts_with(&[46, 0]) || first_bytes.starts_with(&[46, 46, 0]) { + None + } else { + Some(DirEntry { + root: root.clone(), + data: *wfd, + }) + } + } + + fn path(&self) -> PathBuf { + self.root.join(&self.file_name()) + } + + fn file_name(&self) -> OsString { + let filename = truncate_utf16_at_nul(&self.data.cFileName); + OsString::from_wide(filename) + } + + fn file_type(&self) -> io::Result { + Ok(FileType::new( + self.data.dwFileAttributes, + /* reparse_tag = */ self.data.dwReserved0, + )) + } + + fn metadata(&self) -> io::Result { + Ok(FileAttr { + attributes: self.data.dwFileAttributes, + creation_time: self.data.ftCreationTime, + last_access_time: self.data.ftLastAccessTime, + last_write_time: self.data.ftLastWriteTime, + file_size: ((self.data.nFileSizeHigh as u64) << 32) + | (self.data.nFileSizeLow as u64), + reparse_tag: if self.data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + // reserved unless this is a reparse point + self.data.dwReserved0 + } else { + 0 + }, + }) + } + } + + struct DirEntry { + root: Arc, + data: WIN32_FIND_DATAW, + } + + struct ReadDir { + handle: FindNextFileHandle, + root: Arc, + first: Option, + } + + impl Iterator for ReadDir { + type Item = io::Result; + fn next(&mut self) -> Option> { + if let Some(first) = self.first.take() { + if let Some(e) = DirEntry::new(&self.root, &first) { + return Some(Ok(e)); + } + } + unsafe { + let mut wfd = mem::zeroed(); + loop { + if FindNextFileW(self.handle.0, &mut wfd) == 0 { + if GetLastError() == ERROR_NO_MORE_FILES { + return None; + } else { + return Some(Err(io::Error::last_os_error())); + } + } + if let Some(e) = DirEntry::new(&self.root, &wfd) { + return Some(Ok(e)); + } + } + } + } + } + + #[derive(Clone)] + struct FileAttr { + attributes: DWORD, + creation_time: FILETIME, + last_access_time: FILETIME, + last_write_time: FILETIME, + file_size: u64, + reparse_tag: DWORD, + } + + impl FileAttr { + fn perm(&self) -> FilePermissions { + FilePermissions { + readonly: self.attributes & FILE_ATTRIBUTE_READONLY != 0, + } + } + + fn file_type(&self) -> FileType { + FileType::new(self.attributes, self.reparse_tag) + } + + fn is_reparse_point(&self) -> bool { + self.attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + } + } + + #[repr(C)] + struct REPARSE_DATA_BUFFER { + ReparseTag: c_uint, + ReparseDataLength: c_ushort, + Reserved: c_ushort, + rest: (), + } + + const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: usize = 16 * 1024; + + /// An owned container for `HANDLE` object, closing them on Drop. + /// + /// All methods are inherited through a `Deref` impl to `RawHandle` + struct Handle(RawHandle); + + use std::ops::Deref; + + /// A wrapper type for `HANDLE` objects to give them proper Send/Sync inference + /// as well as Rust-y methods. + /// + /// This does **not** drop the handle when it goes out of scope, use `Handle` + /// instead for that. + #[derive(Copy, Clone)] + struct RawHandle(HANDLE); + + unsafe impl Send for RawHandle {} + unsafe impl Sync for RawHandle {} + + impl Handle { + fn new(handle: HANDLE) -> Handle { + Handle(RawHandle::new(handle)) + } + } + + impl Deref for Handle { + type Target = RawHandle; + fn deref(&self) -> &RawHandle { + &self.0 + } + } + + impl Drop for Handle { + fn drop(&mut self) { + unsafe { + let _ = CloseHandle(self.raw()); + } + } + } + + impl RawHandle { + fn new(handle: HANDLE) -> RawHandle { + RawHandle(handle) + } + + fn raw(&self) -> HANDLE { + self.0 + } + } + + struct FindNextFileHandle(HANDLE); + + fn get_path(f: &File) -> io::Result { + fill_utf16_buf( + |buf, sz| unsafe { + GetFinalPathNameByHandleW(f.handle.raw(), buf, sz, VOLUME_NAME_DOS) + }, + |buf| PathBuf::from(OsString::from_wide(buf)), + ) + } + + fn move_item(file: &File, ctx: &mut RmdirContext) -> io::Result<()> { + let mut tmpname = ctx.base_dir.join(format! {"rm-{}", ctx.counter}); + ctx.counter += 1; + // Try to rename the file. If it already exists, just retry with an other + // filename. + while let Err(err) = file.rename(tmpname.as_ref(), false) { + if err.kind() != io::ErrorKind::AlreadyExists { + return Err(err); + }; + tmpname = ctx.base_dir.join(format!("rm-{}", ctx.counter)); + ctx.counter += 1; + } + Ok(()) + } + + fn set_perm(path: &Path, perm: FilePermissions) -> io::Result<()> { + let mut opts = OpenOptions::new(); + opts.access_mode(FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES); + opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS); + let file = File::open(path, &opts)?; + file.set_perm(perm) + } + + const VOLUME_NAME_DOS: DWORD = 0x0; +} diff --git a/src/tools/rust-installer/src/scripter.rs b/src/tools/rust-installer/src/scripter.rs new file mode 100644 index 0000000000000..06affc029fd1e --- /dev/null +++ b/src/tools/rust-installer/src/scripter.rs @@ -0,0 +1,68 @@ +use crate::util::*; +use anyhow::{Context, Result}; +use std::io::Write; + +const TEMPLATE: &'static str = include_str!("../install-template.sh"); + +actor! { + #[derive(Debug)] + pub struct Scripter { + /// The name of the product, for display + #[clap(value_name = "NAME")] + product_name: String = "Product", + + /// The directory under lib/ where the manifest lives + #[clap(value_name = "DIR")] + rel_manifest_dir: String = "manifestlib", + + /// The string to print after successful installation + #[clap(value_name = "MESSAGE")] + success_message: String = "Installed.", + + /// Places to look for legacy manifests to uninstall + #[clap(value_name = "DIRS")] + legacy_manifest_dirs: String = "", + + /// The name of the output script + #[clap(value_name = "FILE")] + output_script: String = "install.sh", + } +} + +impl Scripter { + /// Generates the actual installer script + pub fn run(self) -> Result<()> { + // Replace dashes in the success message with spaces (our arg handling botches spaces) + // TODO: still needed? Kept for compatibility for now. + let product_name = self.product_name.replace('-', " "); + + // Replace dashes in the success message with spaces (our arg handling botches spaces) + // TODO: still needed? Kept for compatibility for now. + let success_message = self.success_message.replace('-', " "); + + let script = TEMPLATE + .replace("%%TEMPLATE_PRODUCT_NAME%%", &sh_quote(&product_name)) + .replace("%%TEMPLATE_REL_MANIFEST_DIR%%", &self.rel_manifest_dir) + .replace("%%TEMPLATE_SUCCESS_MESSAGE%%", &sh_quote(&success_message)) + .replace( + "%%TEMPLATE_LEGACY_MANIFEST_DIRS%%", + &sh_quote(&self.legacy_manifest_dirs), + ) + .replace( + "%%TEMPLATE_RUST_INSTALLER_VERSION%%", + &sh_quote(&crate::RUST_INSTALLER_VERSION), + ); + + create_new_executable(&self.output_script)? + .write_all(script.as_ref()) + .with_context(|| format!("failed to write output script '{}'", self.output_script))?; + + Ok(()) + } +} + +fn sh_quote(s: &T) -> String { + // We'll single-quote the whole thing, so first replace single-quotes with + // '"'"' (leave quoting, double-quote one `'`, re-enter single-quoting) + format!("'{}'", s.to_string().replace('\'', r#"'"'"'"#)) +} diff --git a/src/tools/rust-installer/src/tarballer.rs b/src/tools/rust-installer/src/tarballer.rs new file mode 100644 index 0000000000000..76f5af3fa5392 --- /dev/null +++ b/src/tools/rust-installer/src/tarballer.rs @@ -0,0 +1,143 @@ +use anyhow::{bail, Context, Result}; +use std::fs::{read_link, symlink_metadata}; +use std::io::{empty, BufWriter, Write}; +use std::path::Path; +use tar::{Builder, Header}; +use walkdir::WalkDir; + +use crate::{ + compression::{CombinedEncoder, CompressionFormats}, + util::*, +}; + +actor! { + #[derive(Debug)] + pub struct Tarballer { + /// The input folder to be compressed. + #[clap(value_name = "NAME")] + input: String = "package", + + /// The prefix of the tarballs. + #[clap(value_name = "PATH")] + output: String = "./dist", + + /// The folder in which the input is to be found. + #[clap(value_name = "DIR")] + work_dir: String = "./workdir", + + /// The formats used to compress the tarball. + #[clap(value_name = "FORMAT", default_value_t)] + compression_formats: CompressionFormats, + } +} + +impl Tarballer { + /// Generates the actual tarballs + pub fn run(self) -> Result<()> { + let tarball_name = self.output.clone() + ".tar"; + let encoder = CombinedEncoder::new( + self.compression_formats + .iter() + .map(|f| f.encode(&tarball_name)) + .collect::>>()?, + ); + + // Sort files by their suffix, to group files with the same name from + // different locations (likely identical) and files with the same + // extension (likely containing similar data). + let (dirs, mut files) = get_recursive_paths(&self.work_dir, &self.input) + .context("failed to collect file paths")?; + files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); + + // Write the tar into both encoded files. We write all directories + // first, so files may be directly created. (See rust-lang/rustup.rs#1092.) + let buf = BufWriter::with_capacity(1024 * 1024, encoder); + let mut builder = Builder::new(buf); + + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + pool.install(move || { + for path in dirs { + let src = Path::new(&self.work_dir).join(&path); + builder + .append_dir(&path, &src) + .with_context(|| format!("failed to tar dir '{}'", src.display()))?; + } + for path in files { + let src = Path::new(&self.work_dir).join(&path); + append_path(&mut builder, &src, &path) + .with_context(|| format!("failed to tar file '{}'", src.display()))?; + } + builder + .into_inner() + .context("failed to finish writing .tar stream")? + .into_inner() + .ok() + .unwrap() + .finish()?; + + Ok(()) + }) + } +} + +fn append_path(builder: &mut Builder, src: &Path, path: &String) -> Result<()> { + let stat = symlink_metadata(src)?; + let mut header = Header::new_gnu(); + header.set_metadata(&stat); + if stat.file_type().is_symlink() { + let link = read_link(src)?; + header.set_link_name(&link)?; + builder.append_data(&mut header, path, &mut empty())?; + } else { + if cfg!(windows) { + // Windows doesn't really have a mode, so `tar` never marks files executable. + // Use an extension whitelist to update files that usually should be so. + const EXECUTABLES: [&'static str; 4] = ["exe", "dll", "py", "sh"]; + if let Some(ext) = src.extension().and_then(|s| s.to_str()) { + if EXECUTABLES.contains(&ext) { + let mode = header.mode()?; + header.set_mode(mode | 0o111); + } + } + } + let file = open_file(src)?; + builder.append_data(&mut header, path, &file)?; + } + Ok(()) +} + +/// Returns all `(directories, files)` under the source path. +fn get_recursive_paths(root: P, name: Q) -> Result<(Vec, Vec)> +where + P: AsRef, + Q: AsRef, +{ + let root = root.as_ref(); + let name = name.as_ref(); + + if !name.is_relative() && !name.starts_with(root) { + bail!( + "input '{}' is not in work dir '{}'", + name.display(), + root.display() + ); + } + + let mut dirs = vec![]; + let mut files = vec![]; + for entry in WalkDir::new(root.join(name)) { + let entry = entry?; + let path = entry.path().strip_prefix(root)?; + let path = path_to_str(&path)?; + + if entry.file_type().is_dir() { + dirs.push(path.to_owned()); + } else { + files.push(path.to_owned()); + } + } + Ok((dirs, files)) +} diff --git a/src/tools/rust-installer/src/util.rs b/src/tools/rust-installer/src/util.rs new file mode 100644 index 0000000000000..674617c657c59 --- /dev/null +++ b/src/tools/rust-installer/src/util.rs @@ -0,0 +1,156 @@ +use anyhow::{format_err, Context, Result}; +use std::fs; +use std::path::Path; +use walkdir::WalkDir; + +// Needed to set the script mode to executable. +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +// FIXME: what about Windows? Are default ACLs executable? + +#[cfg(unix)] +use std::os::unix::fs::symlink as symlink_file; +#[cfg(windows)] +use std::os::windows::fs::symlink_file; + +/// Converts a `&Path` to a UTF-8 `&str`. +pub fn path_to_str(path: &Path) -> Result<&str> { + path.to_str() + .ok_or_else(|| format_err!("path is not valid UTF-8 '{}'", path.display())) +} + +/// Wraps `fs::copy` with a nicer error message. +pub fn copy, Q: AsRef>(from: P, to: Q) -> Result { + if fs::symlink_metadata(&from)?.file_type().is_symlink() { + let link = fs::read_link(&from)?; + symlink_file(link, &to)?; + Ok(0) + } else { + let amt = fs::copy(&from, &to).with_context(|| { + format!( + "failed to copy '{}' to '{}'", + from.as_ref().display(), + to.as_ref().display() + ) + })?; + Ok(amt) + } +} + +/// Wraps `fs::create_dir` with a nicer error message. +pub fn create_dir>(path: P) -> Result<()> { + fs::create_dir(&path) + .with_context(|| format!("failed to create dir '{}'", path.as_ref().display()))?; + Ok(()) +} + +/// Wraps `fs::create_dir_all` with a nicer error message. +pub fn create_dir_all>(path: P) -> Result<()> { + fs::create_dir_all(&path) + .with_context(|| format!("failed to create dir '{}'", path.as_ref().display()))?; + Ok(()) +} + +/// Wraps `fs::OpenOptions::create_new().open()` as executable, with a nicer error message. +pub fn create_new_executable>(path: P) -> Result { + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o755); + let file = options + .open(&path) + .with_context(|| format!("failed to create file '{}'", path.as_ref().display()))?; + Ok(file) +} + +/// Wraps `fs::OpenOptions::create_new().open()`, with a nicer error message. +pub fn create_new_file>(path: P) -> Result { + let file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .with_context(|| format!("failed to create file '{}'", path.as_ref().display()))?; + Ok(file) +} + +/// Wraps `fs::File::open()` with a nicer error message. +pub fn open_file>(path: P) -> Result { + let file = fs::File::open(&path) + .with_context(|| format!("failed to open file '{}'", path.as_ref().display()))?; + Ok(file) +} + +/// Wraps `remove_dir_all` with a nicer error message. +pub fn remove_dir_all>(path: P) -> Result<()> { + remove_dir_all::remove_dir_all(path.as_ref()) + .with_context(|| format!("failed to remove dir '{}'", path.as_ref().display()))?; + Ok(()) +} + +/// Wrap `fs::remove_file` with a nicer error message +pub fn remove_file>(path: P) -> Result<()> { + fs::remove_file(path.as_ref()) + .with_context(|| format!("failed to remove file '{}'", path.as_ref().display()))?; + Ok(()) +} + +/// Copies the `src` directory recursively to `dst`. Both are assumed to exist +/// when this function is called. +pub fn copy_recursive(src: &Path, dst: &Path) -> Result<()> { + copy_with_callback(src, dst, |_, _| Ok(())) +} + +/// Copies the `src` directory recursively to `dst`. Both are assumed to exist +/// when this function is called. Invokes a callback for each path visited. +pub fn copy_with_callback(src: &Path, dst: &Path, mut callback: F) -> Result<()> +where + F: FnMut(&Path, fs::FileType) -> Result<()>, +{ + for entry in WalkDir::new(src).min_depth(1) { + let entry = entry?; + let file_type = entry.file_type(); + let path = entry.path().strip_prefix(src)?; + let dst = dst.join(path); + + if file_type.is_dir() { + create_dir(&dst)?; + } else { + copy(entry.path(), dst)?; + } + callback(&path, file_type)?; + } + Ok(()) +} + +macro_rules! actor_field_default { + () => { Default::default() }; + (= $expr:expr) => { $expr.into() } +} + +/// Creates an "actor" with default values, setters for all fields, and Clap parser support. +macro_rules! actor { + ($( #[ $attr:meta ] )+ pub struct $name:ident { + $( $( #[ $field_attr:meta ] )+ $field:ident : $type:ty $(= $default:tt)*, )* + }) => { + $( #[ $attr ] )+ + #[derive(clap::Args)] + pub struct $name { + $( $( #[ $field_attr ] )+ #[clap(long, $(default_value = $default)*)] $field : $type, )* + } + + impl Default for $name { + fn default() -> $name { + $name { + $($field : actor_field_default!($(= $default)*), )* + } + } + } + + impl $name { + $(pub fn $field(&mut self, value: $type) -> &mut Self { + self.$field = value; + self + })* + } + } +} diff --git a/src/tools/rust-installer/test.sh b/src/tools/rust-installer/test.sh new file mode 100755 index 0000000000000..bf6de4cb1fa00 --- /dev/null +++ b/src/tools/rust-installer/test.sh @@ -0,0 +1,1342 @@ +#!/bin/bash + +set -e -u + +if [ -x /bin/echo ]; then + ECHO='/bin/echo' +else + ECHO='echo' +fi + +# Prints the absolute path of a directory to stdout +abs_path() { + local path="$1" + # Unset CDPATH because it causes havok: it makes the destination unpredictable + # and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null + # for good measure. + (unset CDPATH && cd "$path" > /dev/null && pwd) +} + +S="$(abs_path $(dirname $0))" + +TEST_DIR="$S/test" +TMP_DIR="$S/tmp" +WORK_DIR="$TMP_DIR/workdir" +OUT_DIR="$TMP_DIR/outdir" +PREFIX_DIR="$TMP_DIR/prefix" + +case $(uname -s) in + + MINGW* | MSYS*) + WINDOWS=1 + ;; +esac + +say() { + echo "test: $1" +} + +pre() { + echo "test: $1" + rm -Rf "$WORK_DIR" + rm -Rf "$OUT_DIR" + rm -Rf "$PREFIX_DIR" + mkdir -p "$WORK_DIR" + mkdir -p "$OUT_DIR" + mkdir -p "$PREFIX_DIR" +} + +need_ok() { + if [ $? -ne 0 ] + then + echo + echo "TEST FAILED!" + echo + exit 1 + fi +} + +fail() { + echo + echo "$1" + echo + echo "TEST FAILED!" + echo + exit 1 +} + +try() { + set +e + _cmd="$@" + _output=`$@ 2>&1` + if [ $? -ne 0 ]; then + echo \$ "$_cmd" + # Using /bin/echo to avoid escaping + $ECHO "$_output" + echo + echo "TEST FAILED!" + echo + exit 1 + else + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_CMD-}" ]; then + echo \$ "$_cmd" + fi + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_OUTPUT-}" ]; then + $ECHO "$_output" + fi + fi + set -e +} + +expect_fail() { + set +e + _cmd="$@" + _output=`$@ 2>&1` + if [ $? -eq 0 ]; then + echo \$ "$_cmd" + # Using /bin/echo to avoid escaping + $ECHO "$_output" + echo + echo "TEST FAILED!" + echo + exit 1 + else + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_CMD-}" ]; then + echo \$ "$_cmd" + fi + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_OUTPUT-}" ]; then + $ECHO "$_output" + fi + fi + set -e +} + +expect_output_ok() { + set +e + local _expected="$1" + shift 1 + _cmd="$@" + _output=`$@ 2>&1` + if [ $? -ne 0 ]; then + echo \$ "$_cmd" + # Using /bin/echo to avoid escaping + $ECHO "$_output" + echo + echo "TEST FAILED!" + echo + exit 1 + elif ! echo "$_output" | grep -q "$_expected"; then + echo \$ "$_cmd" + $ECHO "$_output" + echo + echo "missing expected output '$_expected'" + echo + echo + echo "TEST FAILED!" + echo + exit 1 + else + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_CMD-}" ]; then + echo \$ "$_cmd" + fi + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_OUTPUT-}" ]; then + $ECHO "$_output" + fi + fi + set -e +} + +expect_output_fail() { + set +e + local _expected="$1" + shift 1 + _cmd="$@" + _output=`$@ 2>&1` + if [ $? -eq 0 ]; then + echo \$ "$_cmd" + # Using /bin/echo to avoid escaping + $ECHO "$_output" + echo + echo "TEST FAILED!" + echo + exit 1 + elif ! echo "$_output" | grep -q "$_expected"; then + echo \$ "$_cmd" + $ECHO "$_output" + echo + echo "missing expected output '$_expected'" + echo + echo + echo "TEST FAILED!" + echo + exit 1 + else + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_CMD-}" ]; then + echo \$ "$_cmd" + fi + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_OUTPUT-}" ]; then + $ECHO "$_output" + fi + fi + set -e +} + +expect_not_output_ok() { + set +e + local _expected="$1" + shift 1 + _cmd="$@" + _output=`$@ 2>&1` + if [ $? -ne 0 ]; then + echo \$ "$_cmd" + # Using /bin/echo to avoid escaping + $ECHO "$_output" + echo + echo "TEST FAILED!" + echo + exit 1 + elif echo "$_output" | grep -q "$_expected"; then + echo \$ "$_cmd" + $ECHO "$_output" + echo + echo "unexpected output '$_expected'" + echo + echo + echo "TEST FAILED!" + echo + exit 1 + else + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_CMD-}" ]; then + echo \$ "$_cmd" + fi + if [ -n "${VERBOSE-}" -o -n "${VERBOSE_OUTPUT-}" ]; then + $ECHO "$_output" + fi + fi + set -e +} + +runtest() { + local _testname="$1" + if [ -n "${TESTNAME-}" ]; then + if ! echo "$_testname" | grep -q "$TESTNAME"; then + return 0 + fi + fi + + pre "$_testname" + "$_testname" +} + +# Installation tests + +basic_install() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/something-to-install" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/program2" + try test -e "$PREFIX_DIR/bin/bad-bin" +} +runtest basic_install + +basic_uninstall() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/package/install.sh --uninstall" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/something-to-install" + try test ! -e "$PREFIX_DIR/dir-to-install/foo" + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/program2" + try test ! -e "$PREFIX_DIR/bin/bad-bin" + try test ! -e "$PREFIX_DIR/lib/packagelib" +} +runtest basic_uninstall + +not_installed_files() { + mkdir -p "$WORK_DIR/overlay" + touch "$WORK_DIR/overlay/not-installed" + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --non-installed-overlay="$WORK_DIR/overlay" + try test -e "$WORK_DIR/package/not-installed" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/not-installed" +} +runtest not_installed_files + +tarball_with_package_name() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc-nightly + try "$WORK_DIR/rustc-nightly/install.sh" --prefix="$PREFIX_DIR" + try test -e "$OUT_DIR/rustc-nightly.tar.gz" + try test -e "$OUT_DIR/rustc-nightly.tar.xz" +} +runtest tarball_with_package_name + +install_overwrite_backup() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try mkdir -p "$PREFIX_DIR/bin" + touch "$PREFIX_DIR/bin/program" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + # The existing program was backed up by 'install' + try test -e "$PREFIX_DIR/bin/program.old" +} +runtest install_overwrite_backup + +bulk_directory() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --bulk-dirs=dir-to-install + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/something-to-install" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/program2" + try test -e "$PREFIX_DIR/bin/bad-bin" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" --uninstall + try test ! -e "$PREFIX_DIR/dir-to-install" +} +runtest bulk_directory + +bulk_directory_overwrite() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --bulk-dirs=dir-to-install + try mkdir -p "$PREFIX_DIR/dir-to-install" + try touch "$PREFIX_DIR/dir-to-install/overwrite" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + # The file that used to exist in the directory no longer does + try test ! -e "$PREFIX_DIR/dir-to-install/overwrite" + # It was backed up + try test -e "$PREFIX_DIR/dir-to-install.old/overwrite" +} +runtest bulk_directory_overwrite + +bulk_directory_overwrite_existing_backup() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --bulk-dirs=dir-to-install + try mkdir -p "$PREFIX_DIR/dir-to-install" + try touch "$PREFIX_DIR/dir-to-install/overwrite" + # This time we've already got an existing backup of the overwritten directory. + # The install should still succeed. + try mkdir -p "$PREFIX_DIR/dir-to-install~" + try touch "$PREFIX_DIR/dir-to-install~/overwrite" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/dir-to-install/overwrite" + try test -e "$PREFIX_DIR/dir-to-install~/overwrite" +} +runtest bulk_directory_overwrite_existing_backup + +nested_bulk_directory() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --bulk-dirs=dir-to-install/qux + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/dir-to-install/qux/bar" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" --uninstall + try test ! -e "$PREFIX_DIR/dir-to-install/qux" +} +runtest nested_bulk_directory + +only_bulk_directory_no_files() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image5" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --bulk-dirs=dir-to-install + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" --uninstall + try test ! -e "$PREFIX_DIR/dir-to-install/foo" +} +runtest only_bulk_directory_no_files + +nested_not_installed_files() { + mkdir -p "$WORK_DIR/overlay" + touch "$WORK_DIR/overlay/not-installed" + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --non-installed-overlay="$WORK_DIR/overlay" + try test -e "$WORK_DIR/package/not-installed" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/not-installed" +} +runtest nested_not_installed_files + +multiple_components() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR/c1" \ + --output-dir="$OUT_DIR/c1" \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR/c2" \ + --output-dir="$OUT_DIR/c2" \ + --component-name=cargo + try "$WORK_DIR/c1/package/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/c2/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/something-to-install" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/program2" + try test -e "$PREFIX_DIR/bin/bad-bin" + try test -e "$PREFIX_DIR/bin/cargo" + try "$WORK_DIR/c1/package/install.sh" --prefix="$PREFIX_DIR" --uninstall + try test ! -e "$PREFIX_DIR/something-to-install" + try test ! -e "$PREFIX_DIR/dir-to-install/foo" + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/program2" + try test ! -e "$PREFIX_DIR/bin/bad-bin" + try "$WORK_DIR/c2/package/install.sh" --prefix="$PREFIX_DIR" --uninstall + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/lib/packagelib" +} +runtest multiple_components + +uninstall_from_installed_script() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR/c1" \ + --output-dir="$OUT_DIR/c1" \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR/c2" \ + --output-dir="$OUT_DIR/c2" \ + --component-name=cargo + try "$WORK_DIR/c1/package/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/c2/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/something-to-install" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/program2" + try test -e "$PREFIX_DIR/bin/bad-bin" + try test -e "$PREFIX_DIR/bin/cargo" + # All components should be uninstalled by this script + try sh "$PREFIX_DIR/lib/packagelib/uninstall.sh" + try test ! -e "$PREFIX_DIR/something-to-install" + try test ! -e "$PREFIX_DIR/dir-to-install/foo" + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/program2" + try test ! -e "$PREFIX_DIR/bin/bad-bin" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/lib/packagelib" +} +runtest uninstall_from_installed_script + +uninstall_from_installed_script_with_args_fails() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR/c1" \ + --output-dir="$OUT_DIR/c1" \ + --component-name=rustc + try "$WORK_DIR/c1/package/install.sh" --prefix="$PREFIX_DIR" + expect_output_fail "uninstall.sh does not take any arguments" sh "$PREFIX_DIR/lib/packagelib/uninstall.sh" --prefix=foo +} +runtest uninstall_from_installed_script_with_args_fails + +# Combined installer tests + +combine_installers() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/something-to-install" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/program2" + try test -e "$PREFIX_DIR/bin/bad-bin" + try test -e "$PREFIX_DIR/bin/cargo" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/something-to-install" + try test ! -e "$PREFIX_DIR/dir-to-install/foo" + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/program2" + try test ! -e "$PREFIX_DIR/bin/bad-bin" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/lib/packagelib" +} +runtest combine_installers + +combine_three_installers() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/something-to-install" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/program2" + try test -e "$PREFIX_DIR/bin/bad-bin" + try test -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/dir-to-install/qux/bar" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/something-to-install" + try test ! -e "$PREFIX_DIR/dir-to-install/foo" + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/program2" + try test ! -e "$PREFIX_DIR/bin/bad-bin" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/lib/packagelib" + try test ! -e "$PREFIX_DIR/dir-to-install/qux/bar" +} +runtest combine_three_installers + +combine_installers_with_overlay() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + mkdir -p "$WORK_DIR/overlay" + touch "$WORK_DIR/overlay/README" + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" \ + --non-installed-overlay="$WORK_DIR/overlay" + try test -e "$WORK_DIR/rust/README" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/README" +} +runtest combine_installers_with_overlay + +combined_with_bulk_dirs() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc \ + --bulk-dirs=dir-to-install + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/dir-to-install" +} +runtest combined_with_bulk_dirs + +combine_install_with_separate_uninstall() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc \ + --rel-manifest-dir=rustlib + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo \ + --rel-manifest-dir=rustlib + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" \ + --rel-manifest-dir=rustlib + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/something-to-install" + try test -e "$PREFIX_DIR/dir-to-install/foo" + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/program2" + try test -e "$PREFIX_DIR/bin/bad-bin" + try test -e "$PREFIX_DIR/bin/cargo" + try "$WORK_DIR/rustc/install.sh --uninstall" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/something-to-install" + try test ! -e "$PREFIX_DIR/dir-to-install/foo" + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/program2" + try test ! -e "$PREFIX_DIR/bin/bad-bin" + try "$WORK_DIR/cargo/install.sh --uninstall" --prefix="$PREFIX_DIR" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/lib/packagelib" +} +runtest combine_install_with_separate_uninstall + +select_components_to_install() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --components=rustc + try test -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --components=cargo + try test ! -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --components=rust-docs + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --components=rustc,cargo + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --components=rustc,cargo,rust-docs + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try test ! -e "$PREFIX_DIR/lib/packagelib" +} +runtest select_components_to_install + +select_components_to_uninstall() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --components=rustc + try test ! -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --components=cargo + try test -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --components=rust-docs + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --components=rustc,cargo + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --components=rustc,cargo,rust-docs + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try test ! -e "$PREFIX_DIR/lib/packagelib" +} +runtest select_components_to_uninstall + +invalid_component() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + expect_output_fail "unknown component" "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --components=foo +} +runtest invalid_component + +without_components() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --without=rust-docs + try test -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --without=rust-docs,cargo + try test -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --without=rust-docs,rustc + try test ! -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/cargo" + try test ! -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" +} +runtest without_components + +# --uninstall --without is kind of weird, +# --without causes components to remain installed +uninstall_without_components() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --without=rust-docs + try test ! -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --without=rust-docs,cargo + try test ! -e "$PREFIX_DIR/bin/program" + try test -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/baz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + try "$WORK_DIR/rust/install.sh --uninstall" --prefix="$PREFIX_DIR" --without=rust-docs,rustc + try test -e "$PREFIX_DIR/bin/program" + try test ! -e "$PREFIX_DIR/bin/cargo" + try test -e "$PREFIX_DIR/baz" +} +runtest uninstall_without_components + +without_any_components() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + expect_output_fail "no components selected for installation" \ + "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --without=rust-docs,rustc,cargo +} +runtest without_any_components + +uninstall_without_any_components() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" + expect_output_fail "no components selected for uninstallation" \ + "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" \ + --uninstall --without=rust-docs,rustc,cargo +} +runtest uninstall_without_any_components + +list_components() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + expect_output_ok "rustc" "$WORK_DIR/rust/install.sh" --list-components + expect_output_ok "cargo" "$WORK_DIR/rust/install.sh" --list-components + expect_output_ok "rust-docs" "$WORK_DIR/rust/install.sh" --list-components +} +runtest list_components + +combined_remains() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image4" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust-docs \ + --component-name=rust-docs + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" + for component in rustc cargo rust-docs; do + # rustbuild wants the original extracted package intact too + try test -d "$WORK_DIR/$component/$component" + try test -d "$WORK_DIR/rust/$component" + done +} +runtest combined_remains + +# Smoke tests + +cannot_write_error() { + # chmod doesn't work on windows + if [ ! -n "${WINDOWS-}" ]; then + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + chmod u-w "$PREFIX_DIR" + expect_fail "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + chmod u+w "$PREFIX_DIR" + fi +} +runtest cannot_write_error + +cannot_install_to_installer() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=my-package + expect_output_fail "cannot install to same directory as installer" \ + "$WORK_DIR/my-package/install.sh" --prefix="$WORK_DIR/my-package" +} +runtest cannot_install_to_installer + +upgrade_from_future_installer_error() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --rel-manifest-dir=rustlib + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + echo 100 > "$PREFIX_DIR/lib/rustlib/rust-installer-version" + expect_fail "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" +} +runtest upgrade_from_future_installer_error + +destdir() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --destdir="$PREFIX_DIR/" --prefix=prefix + try test -e "$PREFIX_DIR/prefix/bin/program" +} +runtest destdir + +destdir_no_trailing_slash() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --destdir="$PREFIX_DIR" --prefix=prefix + try test -e "$PREFIX_DIR/prefix/bin/program" +} +runtest destdir_no_trailing_slash + +disable_verify_noop() { + # Obsolete --disable-verify flag doesn't generate error + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" --disable-verify +} +runtest disable_verify_noop + +create_log() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/lib/packagelib/install.log" + local _log="$(cat "$PREFIX_DIR/lib/packagelib/install.log")" + if [ -z "$_log" ]; then + fail "log is empty" + fi +} +runtest create_log + +leave_log_after_failure() { + # chmod doesn't work on windows + if [ ! -n "${WINDOWS-}" ]; then + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + mkdir -p "$PREFIX_DIR/lib/packagelib" + touch "$PREFIX_DIR/lib/packagelib/components" + chmod u-w "$PREFIX_DIR/lib/packagelib/components" + expect_fail "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + chmod u+w "$PREFIX_DIR/lib/packagelib/components" + try test -e "$PREFIX_DIR/lib/packagelib/install.log" + local _log="$(cat "$PREFIX_DIR/lib/packagelib/install.log")" + if [ -z "$_log" ]; then + fail "log is empty" + fi + # script should tell user where the logs are + if ! grep -q "see logs at" "$PREFIX_DIR/lib/packagelib/install.log"; then + fail "missing log message" + fi + fi +} +runtest leave_log_after_failure + +# https://github.com/rust-lang/rust-installer/issues/22 +help() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --help +} +runtest help + +# https://github.com/rust-lang/rust-installer/issues/31 +CDPATH_does_not_destroy_things() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + cd "$WORK_DIR" || exit 1 + export CDPATH="../$(basename $WORK_DIR)/foo" + try sh "package/install.sh" --prefix="$PREFIX_DIR" + cd "$S" || exit 1 + cd "$PREFIX_DIR" || exit 1 + export CDPATH="../$(basename $PREFIX_DIR)" + try sh "lib/packagelib/uninstall.sh" + cd "$S" || exit 1 + unset CDPATH +} +runtest CDPATH_does_not_destroy_things + +docdir_default() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image-docdir1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" + try test -e "$PREFIX_DIR/share/doc/rust/README" + try test -e "$PREFIX_DIR/share/doc/rust/rustdocs.txt" +} +runtest docdir_default + +docdir() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image-docdir1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" + try mkdir "$WORK_DIR/docdir" + try "$WORK_DIR/package/install.sh" --prefix="$PREFIX_DIR" --docdir="$WORK_DIR/docdir" + try test -e "$WORK_DIR/docdir/README" + try test -e "$WORK_DIR/docdir/rustdocs.txt" +} +runtest docdir + +docdir_combined() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image-docdir1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name="rustc" \ + --component-name="rustc" + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image-docdir2" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name="cargo" \ + --component-name="cargo" + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" + try mkdir "$WORK_DIR/docdir" + try "$WORK_DIR/rust/install.sh" --prefix="$PREFIX_DIR" --docdir="$WORK_DIR/docdir" + try test -e "$WORK_DIR/docdir/README" + try test -e "$WORK_DIR/docdir/rustdocs.txt" + try test -e "$WORK_DIR/docdir/README" + try test -e "$WORK_DIR/docdir/cargodocs.txt" +} +runtest docdir_combined + +combine_installers_different_input_compression_formats() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc \ + --compression-formats=xz + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo \ + --compression-formats=gz + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.xz,$OUT_DIR/cargo.tar.gz" + + try test -e "${OUT_DIR}/rust.tar.gz" + try test -e "${OUT_DIR}/rust.tar.xz" +} +runtest combine_installers_different_input_compression_formats + +generate_compression_formats_one() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name="rustc" \ + --component-name="rustc" \ + --compression-formats="xz" + + try test ! -e "${OUT_DIR}/rustc.tar.gz" + try test -e "${OUT_DIR}/rustc.tar.xz" +} +runtest generate_compression_formats_one + +generate_compression_formats_multiple() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name="rustc" \ + --component-name="rustc" \ + --compression-formats="gz,xz" + + try test -e "${OUT_DIR}/rustc.tar.gz" + try test -e "${OUT_DIR}/rustc.tar.xz" +} +runtest generate_compression_formats_multiple + +generate_compression_formats_error() { + expect_fail sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name="rustc" \ + --component-name="rustc" \ + --compression-formats="xz,foobar" +} +runtest generate_compression_formats_error + +combine_compression_formats_one() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" \ + --compression-formats=xz + + try test ! -e "${OUT_DIR}/rust.tar.gz" + try test -e "${OUT_DIR}/rust.tar.xz" +} +runtest combine_compression_formats_one + +combine_compression_formats_multiple() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + try sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" \ + --compression-formats=xz,gz + + try test -e "${OUT_DIR}/rust.tar.gz" + try test -e "${OUT_DIR}/rust.tar.xz" +} +runtest combine_compression_formats_multiple + +combine_compression_formats_error() { + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image1" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rustc \ + --component-name=rustc + try sh "$S/gen-installer.sh" \ + --image-dir="$TEST_DIR/image3" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=cargo \ + --component-name=cargo + expect_fail sh "$S/combine-installers.sh" \ + --work-dir="$WORK_DIR" \ + --output-dir="$OUT_DIR" \ + --package-name=rust \ + --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz" \ + --compression-formats=xz,foobar +} +runtest combine_compression_formats_error + +tarball_compression_formats_one() { + try cp -r "${TEST_DIR}/image1" "${WORK_DIR}/image" + try sh "$S/make-tarballs.sh" \ + --input="${WORK_DIR}/image" \ + --work-dir="${WORK_DIR}" \ + --output="${OUT_DIR}/rustc" \ + --compression-formats="xz" + + try test ! -e "${OUT_DIR}/rustc.tar.gz" + try test -e "${OUT_DIR}/rustc.tar.xz" +} +runtest tarball_compression_formats_one + +tarball_compression_formats_multiple() { + try cp -r "${TEST_DIR}/image1" "${WORK_DIR}/image" + try sh "$S/make-tarballs.sh" \ + --input="${WORK_DIR}/image" \ + --work-dir="${WORK_DIR}" \ + --output="${OUT_DIR}/rustc" \ + --compression-formats="xz,gz" + + try test -e "${OUT_DIR}/rustc.tar.gz" + try test -e "${OUT_DIR}/rustc.tar.xz" +} +runtest tarball_compression_formats_multiple + +tarball_compression_formats_error() { + try cp -r "${TEST_DIR}/image1" "${WORK_DIR}/image" + expect_fail sh "$S/make-tarballs.sh" \ + --input="${WORK_DIR}/image" \ + --work-dir="${WORK_DIR}" \ + --output="${OUT_DIR}/rustc" \ + --compression-formats="xz,foobar" +} +runtest tarball_compression_formats_error + +echo +echo "TOTAL SUCCESS!" +echo diff --git a/src/tools/rust-installer/test/image-docdir1/share/doc/rust/README b/src/tools/rust-installer/test/image-docdir1/share/doc/rust/README new file mode 100644 index 0000000000000..871732e64f945 --- /dev/null +++ b/src/tools/rust-installer/test/image-docdir1/share/doc/rust/README @@ -0,0 +1 @@ +rust diff --git a/src/tools/rust-installer/test/image-docdir1/share/doc/rust/rustdocs.txt b/src/tools/rust-installer/test/image-docdir1/share/doc/rust/rustdocs.txt new file mode 100644 index 0000000000000..871732e64f945 --- /dev/null +++ b/src/tools/rust-installer/test/image-docdir1/share/doc/rust/rustdocs.txt @@ -0,0 +1 @@ +rust diff --git a/src/tools/rust-installer/test/image-docdir2/share/doc/cargo/README b/src/tools/rust-installer/test/image-docdir2/share/doc/cargo/README new file mode 100644 index 0000000000000..033a48cafdf50 --- /dev/null +++ b/src/tools/rust-installer/test/image-docdir2/share/doc/cargo/README @@ -0,0 +1 @@ +cargo diff --git a/src/tools/rust-installer/test/image-docdir2/share/doc/cargo/cargodocs.txt b/src/tools/rust-installer/test/image-docdir2/share/doc/cargo/cargodocs.txt new file mode 100644 index 0000000000000..033a48cafdf50 --- /dev/null +++ b/src/tools/rust-installer/test/image-docdir2/share/doc/cargo/cargodocs.txt @@ -0,0 +1 @@ +cargo diff --git a/src/tools/rust-installer/test/image1/bin/bad-bin b/src/tools/rust-installer/test/image1/bin/bad-bin new file mode 100644 index 0000000000000..b5b0e3234b424 --- /dev/null +++ b/src/tools/rust-installer/test/image1/bin/bad-bin @@ -0,0 +1 @@ +#!/bin/bogus \ No newline at end of file diff --git a/src/tools/rust-installer/test/image1/bin/program b/src/tools/rust-installer/test/image1/bin/program new file mode 100755 index 0000000000000..96b4b06ad4163 --- /dev/null +++ b/src/tools/rust-installer/test/image1/bin/program @@ -0,0 +1 @@ +#!/bin/sh \ No newline at end of file diff --git a/src/tools/rust-installer/test/image1/bin/program2 b/src/tools/rust-installer/test/image1/bin/program2 new file mode 100755 index 0000000000000..96b4b06ad4163 --- /dev/null +++ b/src/tools/rust-installer/test/image1/bin/program2 @@ -0,0 +1 @@ +#!/bin/sh \ No newline at end of file diff --git a/src/tools/rust-installer/test/image1/dir-to-install/foo b/src/tools/rust-installer/test/image1/dir-to-install/foo new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image1/dir-to-not-install/foo b/src/tools/rust-installer/test/image1/dir-to-not-install/foo new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image1/something-to-install b/src/tools/rust-installer/test/image1/something-to-install new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image1/something-to-not-install b/src/tools/rust-installer/test/image1/something-to-not-install new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image2/bin/oldprogram b/src/tools/rust-installer/test/image2/bin/oldprogram new file mode 100755 index 0000000000000..96b4b06ad4163 --- /dev/null +++ b/src/tools/rust-installer/test/image2/bin/oldprogram @@ -0,0 +1 @@ +#!/bin/sh \ No newline at end of file diff --git a/src/tools/rust-installer/test/image2/dir-to-install/bar b/src/tools/rust-installer/test/image2/dir-to-install/bar new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image2/something-to-install b/src/tools/rust-installer/test/image2/something-to-install new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image3/bin/cargo b/src/tools/rust-installer/test/image3/bin/cargo new file mode 100755 index 0000000000000..96b4b06ad4163 --- /dev/null +++ b/src/tools/rust-installer/test/image3/bin/cargo @@ -0,0 +1 @@ +#!/bin/sh \ No newline at end of file diff --git a/src/tools/rust-installer/test/image4/baz b/src/tools/rust-installer/test/image4/baz new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image4/dir-to-install/qux/bar b/src/tools/rust-installer/test/image4/dir-to-install/qux/bar new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/tools/rust-installer/test/image5/dir-to-install/foo b/src/tools/rust-installer/test/image5/dir-to-install/foo new file mode 100644 index 0000000000000..e69de29bb2d1d From ddc2d1e8063a346124633649b0d97dd6f8c33c83 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Tue, 28 Feb 2023 21:19:07 -0500 Subject: [PATCH 3/4] Add bootstrap support for rust-installer --- src/bootstrap/builder.rs | 1 + src/bootstrap/lib.rs | 7 +-- src/bootstrap/test.rs | 55 +++++++++++++++++++ .../rust-installer/combine-installers.sh | 2 +- src/tools/rust-installer/gen-installer.sh | 2 +- src/tools/rust-installer/make-tarballs.sh | 2 +- src/tools/rust-installer/test.sh | 1 - 7 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index b33fc02f49c24..786e35f6ddef0 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -711,6 +711,7 @@ impl<'a> Builder<'a> { test::RustdocUi, test::RustdocJson, test::HtmlCheck, + test::RustInstaller, // Run bootstrap close to the end as it's unlikely to fail test::Bootstrap, // Run run-make last, since these won't pass without make on Windows diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index f4abdf1cc5758..89cf77fba7b89 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -482,12 +482,7 @@ impl Build { // Make sure we update these before gathering metadata so we don't get an error about missing // Cargo.toml files. - let rust_submodules = [ - "src/tools/rust-installer", - "src/tools/cargo", - "library/backtrace", - "library/stdarch", - ]; + let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"]; for s in rust_submodules { build.update_submodule(Path::new(s)); } diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index b4f1506dc8f30..f5d680df1133b 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -2695,3 +2695,58 @@ impl Step for LintDocs { }); } } + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustInstaller; + +impl Step for RustInstaller { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Ensure the version placeholder replacement tool builds + fn run(self, builder: &Builder<'_>) { + builder.info("test rust-installer"); + + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/rust-installer", + SourceType::InTree, + &[], + ); + try_run(builder, &mut cargo.into()); + + // We currently don't support running the test.sh script outside linux(?) environments. + // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a + // set of scripts, which will likely allow dropping this if. + if bootstrap_host != "x86_64-unknown-linux-gnu" { + return; + } + + let mut cmd = + std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh")); + let tmpdir = testdir(builder, compiler.host).join("rust-installer"); + let _ = std::fs::remove_dir_all(&tmpdir); + let _ = std::fs::create_dir_all(&tmpdir); + cmd.current_dir(&tmpdir); + cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); + cmd.env("CARGO", &builder.initial_cargo); + cmd.env("RUSTC", &builder.initial_rustc); + cmd.env("TMP_DIR", &tmpdir); + try_run(builder, &mut cmd); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-installer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self); + } +} diff --git a/src/tools/rust-installer/combine-installers.sh b/src/tools/rust-installer/combine-installers.sh index 4931c34ddc27f..bdbaab711394f 100755 --- a/src/tools/rust-installer/combine-installers.sh +++ b/src/tools/rust-installer/combine-installers.sh @@ -21,4 +21,4 @@ abs_path() { } src_dir="$(abs_path $(dirname "$0"))" -cargo run --manifest-path="$src_dir/Cargo.toml" -- combine "$@" +$CARGO run --manifest-path="$src_dir/Cargo.toml" -- combine "$@" diff --git a/src/tools/rust-installer/gen-installer.sh b/src/tools/rust-installer/gen-installer.sh index 198cfe7425534..9a2c3016fee44 100755 --- a/src/tools/rust-installer/gen-installer.sh +++ b/src/tools/rust-installer/gen-installer.sh @@ -21,4 +21,4 @@ abs_path() { } src_dir="$(abs_path $(dirname "$0"))" -cargo run --manifest-path="$src_dir/Cargo.toml" -- generate "$@" +$CARGO run --manifest-path="$src_dir/Cargo.toml" -- generate "$@" diff --git a/src/tools/rust-installer/make-tarballs.sh b/src/tools/rust-installer/make-tarballs.sh index e9f88cc8b7186..6fc823666f14b 100755 --- a/src/tools/rust-installer/make-tarballs.sh +++ b/src/tools/rust-installer/make-tarballs.sh @@ -21,4 +21,4 @@ abs_path() { } src_dir="$(abs_path $(dirname "$0"))" -cargo run --manifest-path="$src_dir/Cargo.toml" -- tarball "$@" +$CARGO run --manifest-path="$src_dir/Cargo.toml" -- tarball "$@" diff --git a/src/tools/rust-installer/test.sh b/src/tools/rust-installer/test.sh index bf6de4cb1fa00..dac6f77ef1401 100755 --- a/src/tools/rust-installer/test.sh +++ b/src/tools/rust-installer/test.sh @@ -20,7 +20,6 @@ abs_path() { S="$(abs_path $(dirname $0))" TEST_DIR="$S/test" -TMP_DIR="$S/tmp" WORK_DIR="$TMP_DIR/workdir" OUT_DIR="$TMP_DIR/outdir" PREFIX_DIR="$TMP_DIR/prefix" From dabafb44e0917709910915dc5b3bc351a519a9e1 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Tue, 28 Feb 2023 21:19:43 -0500 Subject: [PATCH 4/4] Use 3 or 6 compression threads for rust-installer Limit to 3 threads for 32-bit platforms, otherwise we use 6 threads. This avoids exceeding the amount of memory we can allocate on a 32-bit platform. --- src/tools/rust-installer/src/compression.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/tools/rust-installer/src/compression.rs b/src/tools/rust-installer/src/compression.rs index 9b176982d000d..013e05fda5819 100644 --- a/src/tools/rust-installer/src/compression.rs +++ b/src/tools/rust-installer/src/compression.rs @@ -61,13 +61,24 @@ impl CompressionFormat { lzma_ops.literal_context_bits(3); filters.lzma2(&lzma_ops); + + let mut builder = xz2::stream::MtStreamBuilder::new(); + builder.filters(filters); + + // On 32-bit platforms limit ourselves to 3 threads, otherwise we exceed memory + // usage this process can take. In the future we'll likely only do super-fast + // compression in CI and move this heavyweight processing to promote-release (which + // is always 64-bit and can run on big-memory machines) but for now this lets us + // move forward. + if std::mem::size_of::() == 4 { + builder.threads(3); + } else { + builder.threads(6); + } + let compressor = XzEncoder::new_stream( std::io::BufWriter::new(file), - xz2::stream::MtStreamBuilder::new() - .threads(1) - .filters(filters) - .encoder() - .unwrap(), + builder.encoder().unwrap(), ); Box::new(compressor) }