Auto merge of #39728 - eddyb:vendeur-tres-bien, r=alexcrichton

Automate vendoring by invoking cargo-vendor when building src dist tarballs.

This avoids #39633 bringing the `src/vendor` checked into git by #37524, past 200,000 lines of code.

I believe the strategy of having rustbuild run `cargo vendor` during the `dist src` step is sound.

However, the only way to be sure `cargo-vendor` exists is to run `cargo install --force cargo-vendor`, which will recompile it every time (not passing `--force` means you can't tell between "already exists" and "build error"). ~~This is quite suboptimal and I'd like to somehow do it in each `Dockerfile` that would need it.~~

* [ ] Cache `CARGO_HOME` (i.e. `~/.cargo`) between CI runs
  * `bin/cargo-vendor` and the actual caches are the relevant bits
* [x] Do not build `cargo-vendor` all the time
  * ~~Maybe detect `~/.cargo/bin/cargo-vendor` already exists?~~
  * ~~Could also try to build it in a `Dockerfile` but do we have `cargo`/`rustc` there?~~
  * Final solution: check `cargo install --list` for a line starting with `cargo-vendor `

cc @rust-lang/tools

.travis.yml

@@ -47,15 +47,24 @@ matrix:
       install: &osx_install_sccache >
         curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
           tar xJf - -C /usr/local/bin --strip-components=1
+    - env: >
+        RUST_CHECK_TARGET=check
+        RUST_CONFIGURE_ARGS=--build=i686-apple-darwin
+        SRC=.
+      os: osx
+      osx_image: xcode8.2
+      install: *osx_install_sccache
 
     - env: >
-        SCRIPT="./x.py test && ./x.py dist"
+        RUST_CHECK_TARGET=dist
         RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended"
         SRC=.
         DEPLOY=1
       os: osx
       osx_image: xcode8.2
-      install: *osx_install_sccache
+      install: >
+        curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
+          tar xJf - -C /usr/local/bin --strip-components=1 && brew uninstall --ignore-dependencies openssl && brew install openssl --universal --without-test
     - env: >
         RUST_CHECK_TARGET=dist
         RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended"

appveyor.yml

@@ -17,7 +17,6 @@ environment:
 
   # MSVC cargotest
   - MSYS_BITS: 64
-    NO_VENDOR: 1
     RUST_CHECK_TARGET: check-aux
     RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
 

configure

@@ -647,6 +647,7 @@ opt llvm-version-check 1 "check if the LLVM version is supported, build anyway"
 opt codegen-tests 1 "run the src/test/codegen tests"
 opt option-checking 1 "complain about unrecognized options in this configure script"
 opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
+opt locked-deps 0 "force Cargo.lock to be up to date"
 opt vendor 0 "enable usage of vendored Rust crates"
 opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)"
 

src/bootstrap/bootstrap.py

@@ -294,6 +294,8 @@ def build_bootstrap(self):
             raise Exception("no cargo executable found at `%s`" % self.cargo())
         args = [self.cargo(), "build", "--manifest-path",
                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
+        if self.use_locked_deps:
+            args.append("--locked")
         if self.use_vendored_sources:
             args.append("--frozen")
         self.run(args, env)
@@ -455,6 +457,9 @@ def main():
     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
                               'CFG_ENABLE_VENDOR' in rb.config_mk
 
+    rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
+                         'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
+
     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
         if os.environ.get('USER') != os.environ['SUDO_USER']:
             rb.use_vendored_sources = True

src/bootstrap/check.rs

@@ -108,8 +108,12 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) {
 pub fn tidy(build: &Build, host: &str) {
     println!("tidy check ({})", host);
     let compiler = Compiler::new(0, host);
-    build.run(build.tool_cmd(&compiler, "tidy")
-                   .arg(build.src.join("src")));
+    let mut cmd = build.tool_cmd(&compiler, "tidy");
+    cmd.arg(build.src.join("src"));
+    if !build.config.vendor {
+        cmd.arg("--no-vendor");
+    }
+    build.run(&mut cmd);
 }
 
 fn testdir(build: &Build, host: &str) -> PathBuf {
@@ -643,6 +647,7 @@ pub fn distcheck(build: &Build) {
     build.run(&mut cmd);
     build.run(Command::new("./configure")
                      .args(&build.config.configure_args)
+                     .arg("--enable-vendor")
                      .current_dir(&dir));
     build.run(Command::new(build_helper::make(&build.config.build))
                      .arg("check")

src/bootstrap/config.rs

@@ -44,6 +44,7 @@ pub struct Config {
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
+    pub locked_deps: bool,
     pub vendor: bool,
     pub target_config: HashMap<String, Target>,
     pub full_bootstrap: bool,
@@ -145,6 +146,7 @@ struct Build {
     docs: Option<bool>,
     submodules: Option<bool>,
     gdb: Option<String>,
+    locked_deps: Option<bool>,
     vendor: Option<bool>,
     nodejs: Option<String>,
     python: Option<String>,
@@ -294,6 +296,7 @@ impl Config {
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
+        set(&mut config.locked_deps, build.locked_deps);
         set(&mut config.vendor, build.vendor);
         set(&mut config.full_bootstrap, build.full_bootstrap);
         set(&mut config.extended, build.extended);
@@ -440,6 +443,7 @@ impl Config {
                 ("LOCAL_REBUILD", self.local_rebuild),
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
+                ("LOCKED_DEPS", self.locked_deps),
                 ("VENDOR", self.vendor),
                 ("FULL_BOOTSTRAP", self.full_bootstrap),
                 ("EXTENDED", self.extended),

src/bootstrap/config.toml.example

@@ -108,6 +108,10 @@
 # Note that Python 2 is currently required.
 #python = "python2.7"
 
+# Force Cargo to check that Cargo.lock describes the precise dependency
+# set that all the Cargo.toml files create, instead of updating it.
+#locked-deps = false
+
 # Indicate whether the vendored sources are used for Rust dependencies or not
 #vendor = false
 

src/bootstrap/dist.rs

@@ -360,6 +360,8 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
     t!(fs::remove_dir_all(&image));
 }
 
+const CARGO_VENDOR_VERSION: &'static str = "0.1.4";
+
 /// Creates the `rust-src` installer component and the plain source tarball
 pub fn rust_src(build: &Build) {
     println!("Dist src");
@@ -404,13 +406,6 @@ pub fn rust_src(build: &Build) {
             }
         }
 
-        // If we're inside the vendor directory then we need to preserve
-        // everything as Cargo's vendoring support tracks all checksums and we
-        // want to be sure we don't accidentally leave out a file.
-        if spath.contains("vendor") {
-            return true
-        }
-
         let excludes = [
             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
@@ -433,6 +428,29 @@ pub fn rust_src(build: &Build) {
         copy(&build.src.join(item), &dst_src.join(item));
     }
 
+    // Get cargo-vendor installed, if it isn't already.
+    let mut has_cargo_vendor = false;
+    let mut cmd = Command::new(&build.cargo);
+    for line in output(cmd.arg("install").arg("--list")).lines() {
+        has_cargo_vendor |= line.starts_with("cargo-vendor ");
+    }
+    if !has_cargo_vendor {
+        let mut cmd = Command::new(&build.cargo);
+        cmd.arg("install")
+           .arg("--force")
+           .arg("--debug")
+           .arg("--vers").arg(CARGO_VENDOR_VERSION)
+           .arg("cargo-vendor")
+           .env("RUSTC", &build.rustc);
+        build.run(&mut cmd);
+    }
+
+    // Vendor all Cargo dependencies
+    let mut cmd = Command::new(&build.cargo);
+    cmd.arg("vendor")
+       .current_dir(&dst_src.join("src"));
+    build.run(&mut cmd);
+
     // Create source tarball in rust-installer format
     let mut cmd = Command::new("sh");
     cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))

src/bootstrap/lib.rs

@@ -528,6 +528,9 @@ impl Build {
         if self.config.rust_optimize && cmd != "bench" {
             cargo.arg("--release");
         }
+        if self.config.locked_deps {
+            cargo.arg("--locked");
+        }
         if self.config.vendor || self.is_sudo {
             cargo.arg("--frozen");
         }

src/ci/docker/android/Dockerfile

@@ -16,7 +16,9 @@ RUN dpkg --add-architecture i386 && \
   openjdk-9-jre \
   sudo \
   libstdc++6:i386 \
-  xz-utils
+  xz-utils \
+  libssl-dev \
+  pkg-config
 
 WORKDIR /android/
 ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools

src/ci/docker/cross/Dockerfile

@@ -17,7 +17,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gcc-sparc64-linux-gnu \
   libc6-dev-sparc64-cross \
   bzip2 \
-  patch
+  patch \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-arm-linux/Dockerfile

@@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
-  xz-utils
+  xz-utils \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-armv7-aarch64-linux/Dockerfile

@@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
-  xz-utils
+  xz-utils \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-freebsd/Dockerfile

@@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   bzip2 \
   xz-utils \
-  wget
+  wget \
+  libssl-dev \
+  pkg-config
 
 COPY build-toolchain.sh /tmp/
 RUN /tmp/build-toolchain.sh x86_64

src/ci/docker/dist-mips-linux/Dockerfile

@@ -13,7 +13,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gdb \
   xz-utils \
   g++-mips-linux-gnu \
-  g++-mipsel-linux-gnu
+  g++-mipsel-linux-gnu \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-mips64-linux/Dockerfile

@@ -13,7 +13,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gdb \
   xz-utils \
   g++-mips64-linux-gnuabi64 \
-  g++-mips64el-linux-gnuabi64
+  g++-mips64el-linux-gnuabi64 \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-powerpc-linux/Dockerfile

@@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
-  xz-utils
+  xz-utils \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-powerpc64-linux/Dockerfile

@@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
-  xz-utils
+  xz-utils \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-s390x-linux-netbsd/Dockerfile

@@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
-  xz-utils
+  xz-utils \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/dist-x86-linux/Dockerfile

@@ -13,13 +13,14 @@ RUN yum upgrade -y && yum install -y \
       file \
       xz \
       which \
-      pkg-config \
+      pkgconfig \
       wget \
       autoconf \
       gettext
 
 ENV PATH=/rustroot/bin:$PATH
 ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib
+ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 WORKDIR /tmp
 COPY shared.sh build-binutils.sh /tmp/
 

src/ci/docker/i686-gnu/Dockerfile

@@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist
+ENV SCRIPT python2.7 ../x.py test

src/ci/docker/linux-tested-targets/Dockerfile

@@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   xz-utils \
   sudo \
   gdb \
-  patch
+  patch \
+  libssl-dev \
+  pkg-config
 
 WORKDIR /build/
 COPY musl-libunwind-patch.patch build-musl.sh /build/

src/ci/docker/x86_64-gnu-aux/Dockerfile

@@ -25,4 +25,3 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
 ENV RUST_CHECK_TARGET check-aux
-ENV NO_VENDOR 1

src/ci/docker/x86_64-gnu-distcheck/Dockerfile

@@ -11,7 +11,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
-  xz-utils
+  xz-utils \
+  libssl-dev \
+  pkg-config
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

src/ci/docker/x86_64-gnu/Dockerfile

@@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-sanitizers
-ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist
+ENV SCRIPT python2.7 ../x.py test

src/ci/run.sh

@@ -23,6 +23,7 @@ fi
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
+RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps"
 
 # If we're deploying artifacts then we set the release channel, otherwise if
 # we're not deploying then we want to be sure to enable all assertions becauase
@@ -47,13 +48,6 @@ else
   fi
 fi
 
-# We want to enable usage of the `src/vendor` dir as much as possible, but not
-# all test suites have all their deps in there (just the main bootstrap) so we
-# have the ability to disable this flag
-if [ "$NO_VENDOR" = "" ]; then
-  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-vendor"
-fi
-
 set -ex
 
 $SRC/configure $RUST_CONFIGURE_ARGS

src/tools/tidy/src/main.rs

@@ -42,14 +42,18 @@ fn main() {
     let path = env::args_os().skip(1).next().expect("need an argument");
     let path = PathBuf::from(path);
 
+    let args: Vec<String> = env::args().skip(1).collect();
+
     let mut bad = false;
     bins::check(&path, &mut bad);
     style::check(&path, &mut bad);
     errors::check(&path, &mut bad);
     cargo::check(&path, &mut bad);
     features::check(&path, &mut bad);
     pal::check(&path, &mut bad);
-    deps::check(&path, &mut bad);
+    if !args.iter().any(|s| *s == "--no-vendor") {
+        deps::check(&path, &mut bad);
+    }
 
     if bad {
         panic!("some tidy checks failed");