diff --git a/.travis.yml b/.travis.yml index a776f1b1e1041..a3bcc85998bce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,15 +47,24 @@ matrix: install: &osx_install_sccache > curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 | tar xJf - -C /usr/local/bin --strip-components=1 + - env: > + RUST_CHECK_TARGET=check + RUST_CONFIGURE_ARGS=--build=i686-apple-darwin + SRC=. + os: osx + osx_image: xcode8.2 + install: *osx_install_sccache - env: > - SCRIPT="./x.py test && ./x.py dist" + RUST_CHECK_TARGET=dist RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended" SRC=. DEPLOY=1 os: osx osx_image: xcode8.2 - install: *osx_install_sccache + install: > + curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 | + tar xJf - -C /usr/local/bin --strip-components=1 && brew uninstall --ignore-dependencies openssl && brew install openssl --universal --without-test - env: > RUST_CHECK_TARGET=dist RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended" diff --git a/appveyor.yml b/appveyor.yml index 2183d8da95f89..9fff2f3c8b4df 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,6 @@ environment: # MSVC cargotest - MSYS_BITS: 64 - NO_VENDOR: 1 RUST_CHECK_TARGET: check-aux RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc diff --git a/configure b/configure index c751ad9731a7d..e9a0b4c476eb2 100755 --- a/configure +++ b/configure @@ -648,6 +648,7 @@ opt rustbuild 1 "use the rust and cargo based build system" opt codegen-tests 1 "run the src/test/codegen tests" opt option-checking 1 "complain about unrecognized options in this configure script" opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)" +opt locked-deps 0 "force Cargo.lock to be up to date" opt vendor 0 "enable usage of vendored Rust crates" opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)" diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 27255b6910093..92a075ea9dc2e 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -294,6 +294,8 @@ def build_bootstrap(self): raise Exception("no cargo executable found at `%s`" % self.cargo()) args = [self.cargo(), "build", "--manifest-path", os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] + if self.use_locked_deps: + args.append("--locked") if self.use_vendored_sources: args.append("--frozen") self.run(args, env) @@ -440,6 +442,9 @@ def main(): rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \ 'CFG_ENABLE_VENDOR' in rb.config_mk + rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \ + 'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk + if 'SUDO_USER' in os.environ and not rb.use_vendored_sources: if os.environ.get('USER') != os.environ['SUDO_USER']: rb.use_vendored_sources = True diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index cbfbcbe4f0c6e..00758460becc5 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -108,8 +108,12 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) { pub fn tidy(build: &Build, host: &str) { println!("tidy check ({})", host); let compiler = Compiler::new(0, host); - build.run(build.tool_cmd(&compiler, "tidy") - .arg(build.src.join("src"))); + let mut cmd = build.tool_cmd(&compiler, "tidy"); + cmd.arg(build.src.join("src")); + if !build.config.vendor { + cmd.arg("--no-vendor"); + } + build.run(&mut cmd); } fn testdir(build: &Build, host: &str) -> PathBuf { @@ -643,6 +647,7 @@ pub fn distcheck(build: &Build) { build.run(&mut cmd); build.run(Command::new("./configure") .args(&build.config.configure_args) + .arg("--enable-vendor") .current_dir(&dir)); build.run(Command::new(build_helper::make(&build.config.build)) .arg("check") diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 47f505ad37e74..e95416be4b6dc 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -44,6 +44,7 @@ pub struct Config { pub submodules: bool, pub compiler_docs: bool, pub docs: bool, + pub locked_deps: bool, pub vendor: bool, pub target_config: HashMap, pub full_bootstrap: bool, @@ -145,6 +146,7 @@ struct Build { docs: Option, submodules: Option, gdb: Option, + locked_deps: Option, vendor: Option, nodejs: Option, python: Option, @@ -294,6 +296,7 @@ impl Config { set(&mut config.compiler_docs, build.compiler_docs); set(&mut config.docs, build.docs); set(&mut config.submodules, build.submodules); + set(&mut config.locked_deps, build.locked_deps); set(&mut config.vendor, build.vendor); set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); @@ -440,6 +443,7 @@ impl Config { ("LOCAL_REBUILD", self.local_rebuild), ("NINJA", self.ninja), ("CODEGEN_TESTS", self.codegen_tests), + ("LOCKED_DEPS", self.locked_deps), ("VENDOR", self.vendor), ("FULL_BOOTSTRAP", self.full_bootstrap), ("EXTENDED", self.extended), diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 5f4303a728c5c..f95e890f346ee 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -108,6 +108,10 @@ # Note that Python 2 is currently required. #python = "python2.7" +# Force Cargo to check that Cargo.lock describes the precise dependency +# set that all the Cargo.toml files create, instead of updating it. +#locked-deps = false + # Indicate whether the vendored sources are used for Rust dependencies or not #vendor = false diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 2da2892150b4f..52a7c63c9045f 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -360,6 +360,8 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) { t!(fs::remove_dir_all(&image)); } +const CARGO_VENDOR_VERSION: &'static str = "0.1.4"; + /// Creates the `rust-src` installer component and the plain source tarball pub fn rust_src(build: &Build) { println!("Dist src"); @@ -404,13 +406,6 @@ pub fn rust_src(build: &Build) { } } - // If we're inside the vendor directory then we need to preserve - // everything as Cargo's vendoring support tracks all checksums and we - // want to be sure we don't accidentally leave out a file. - if spath.contains("vendor") { - return true - } - let excludes = [ "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules", ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}", @@ -433,6 +428,29 @@ pub fn rust_src(build: &Build) { copy(&build.src.join(item), &dst_src.join(item)); } + // Get cargo-vendor installed, if it isn't already. + let mut has_cargo_vendor = false; + let mut cmd = Command::new(&build.cargo); + for line in output(cmd.arg("install").arg("--list")).lines() { + has_cargo_vendor |= line.starts_with("cargo-vendor "); + } + if !has_cargo_vendor { + let mut cmd = Command::new(&build.cargo); + cmd.arg("install") + .arg("--force") + .arg("--debug") + .arg("--vers").arg(CARGO_VENDOR_VERSION) + .arg("cargo-vendor") + .env("RUSTC", &build.rustc); + build.run(&mut cmd); + } + + // Vendor all Cargo dependencies + let mut cmd = Command::new(&build.cargo); + cmd.arg("vendor") + .current_dir(&dst_src.join("src")); + build.run(&mut cmd); + // Create source tarball in rust-installer format let mut cmd = Command::new("sh"); cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh"))) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index ba6b34343f0b3..7bd611eb53e3c 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -528,6 +528,9 @@ impl Build { if self.config.rust_optimize && cmd != "bench" { cargo.arg("--release"); } + if self.config.locked_deps { + cargo.arg("--locked"); + } if self.config.vendor || self.is_sudo { cargo.arg("--frozen"); } diff --git a/src/ci/docker/android/Dockerfile b/src/ci/docker/android/Dockerfile index e3748af501fb8..ae1bfd9e40a80 100644 --- a/src/ci/docker/android/Dockerfile +++ b/src/ci/docker/android/Dockerfile @@ -16,7 +16,9 @@ RUN dpkg --add-architecture i386 && \ openjdk-9-jre \ sudo \ libstdc++6:i386 \ - xz-utils + xz-utils \ + libssl-dev \ + pkg-config WORKDIR /android/ ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools diff --git a/src/ci/docker/cross/Dockerfile b/src/ci/docker/cross/Dockerfile index 29a5e7bcafb7a..52c5b83f26322 100644 --- a/src/ci/docker/cross/Dockerfile +++ b/src/ci/docker/cross/Dockerfile @@ -17,7 +17,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gcc-sparc64-linux-gnu \ libc6-dev-sparc64-cross \ bzip2 \ - patch + patch \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-arm-linux/Dockerfile b/src/ci/docker/dist-arm-linux/Dockerfile index 217a724fb9a36..e5619564b05d2 100644 --- a/src/ci/docker/dist-arm-linux/Dockerfile +++ b/src/ci/docker/dist-arm-linux/Dockerfile @@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ texinfo \ wget \ - xz-utils + xz-utils \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-armv7-aarch64-linux/Dockerfile b/src/ci/docker/dist-armv7-aarch64-linux/Dockerfile index f26885bbb539a..5b94d5a972704 100644 --- a/src/ci/docker/dist-armv7-aarch64-linux/Dockerfile +++ b/src/ci/docker/dist-armv7-aarch64-linux/Dockerfile @@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ texinfo \ wget \ - xz-utils + xz-utils \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-freebsd/Dockerfile b/src/ci/docker/dist-freebsd/Dockerfile index d824c4041ceeb..c39e298e3ceb0 100644 --- a/src/ci/docker/dist-freebsd/Dockerfile +++ b/src/ci/docker/dist-freebsd/Dockerfile @@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ bzip2 \ xz-utils \ - wget + wget \ + libssl-dev \ + pkg-config COPY build-toolchain.sh /tmp/ RUN /tmp/build-toolchain.sh x86_64 diff --git a/src/ci/docker/dist-mips-linux/Dockerfile b/src/ci/docker/dist-mips-linux/Dockerfile index 38ee95038f646..5d8b3cfeae2a3 100644 --- a/src/ci/docker/dist-mips-linux/Dockerfile +++ b/src/ci/docker/dist-mips-linux/Dockerfile @@ -13,7 +13,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ xz-utils \ g++-mips-linux-gnu \ - g++-mipsel-linux-gnu + g++-mipsel-linux-gnu \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-mips64-linux/Dockerfile b/src/ci/docker/dist-mips64-linux/Dockerfile index c9d89d6287450..8feba12e72203 100644 --- a/src/ci/docker/dist-mips64-linux/Dockerfile +++ b/src/ci/docker/dist-mips64-linux/Dockerfile @@ -13,7 +13,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ xz-utils \ g++-mips64-linux-gnuabi64 \ - g++-mips64el-linux-gnuabi64 + g++-mips64el-linux-gnuabi64 \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-powerpc-linux/Dockerfile b/src/ci/docker/dist-powerpc-linux/Dockerfile index 640bacc54eb30..e5ffbfc090d5a 100644 --- a/src/ci/docker/dist-powerpc-linux/Dockerfile +++ b/src/ci/docker/dist-powerpc-linux/Dockerfile @@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ texinfo \ wget \ - xz-utils + xz-utils \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-powerpc64-linux/Dockerfile b/src/ci/docker/dist-powerpc64-linux/Dockerfile index 624763ef5de67..7d13bc3d438ba 100644 --- a/src/ci/docker/dist-powerpc64-linux/Dockerfile +++ b/src/ci/docker/dist-powerpc64-linux/Dockerfile @@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ texinfo \ wget \ - xz-utils + xz-utils \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile b/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile index 589b5fd530fc7..7f4c6d4647c7d 100644 --- a/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile +++ b/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile @@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ texinfo \ wget \ - xz-utils + xz-utils \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/dist-x86-linux/Dockerfile b/src/ci/docker/dist-x86-linux/Dockerfile index 12e24ff1af6a2..61c23201c2cee 100644 --- a/src/ci/docker/dist-x86-linux/Dockerfile +++ b/src/ci/docker/dist-x86-linux/Dockerfile @@ -13,13 +13,14 @@ RUN yum upgrade -y && yum install -y \ file \ xz \ which \ - pkg-config \ + pkgconfig \ wget \ autoconf \ gettext ENV PATH=/rustroot/bin:$PATH ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib +ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig WORKDIR /tmp COPY shared.sh build-binutils.sh /tmp/ diff --git a/src/ci/docker/i686-gnu/Dockerfile b/src/ci/docker/i686-gnu/Dockerfile index 6583f09be368c..84b98712b3385 100644 --- a/src/ci/docker/i686-gnu/Dockerfile +++ b/src/ci/docker/i686-gnu/Dockerfile @@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu -ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist +ENV SCRIPT python2.7 ../x.py test diff --git a/src/ci/docker/linux-tested-targets/Dockerfile b/src/ci/docker/linux-tested-targets/Dockerfile index 2a43201ed0aeb..feb73bebbdbec 100644 --- a/src/ci/docker/linux-tested-targets/Dockerfile +++ b/src/ci/docker/linux-tested-targets/Dockerfile @@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ sudo \ gdb \ - patch + patch \ + libssl-dev \ + pkg-config WORKDIR /build/ COPY musl-libunwind-patch.patch build-musl.sh /build/ diff --git a/src/ci/docker/x86_64-gnu-aux/Dockerfile b/src/ci/docker/x86_64-gnu-aux/Dockerfile index 0ec0bfd189725..607163ea547a3 100644 --- a/src/ci/docker/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/x86_64-gnu-aux/Dockerfile @@ -25,4 +25,3 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu ENV RUST_CHECK_TARGET check-aux -ENV NO_VENDOR 1 diff --git a/src/ci/docker/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/x86_64-gnu-distcheck/Dockerfile index 57a2c103f1e7d..ad6c0c15f7282 100644 --- a/src/ci/docker/x86_64-gnu-distcheck/Dockerfile +++ b/src/ci/docker/x86_64-gnu-distcheck/Dockerfile @@ -11,7 +11,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ sudo \ gdb \ - xz-utils + xz-utils \ + libssl-dev \ + pkg-config ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \ diff --git a/src/ci/docker/x86_64-gnu/Dockerfile b/src/ci/docker/x86_64-gnu/Dockerfile index e903b6ddc64cd..427498fabb2d7 100644 --- a/src/ci/docker/x86_64-gnu/Dockerfile +++ b/src/ci/docker/x86_64-gnu/Dockerfile @@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-sanitizers -ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist +ENV SCRIPT python2.7 ../x.py test diff --git a/src/ci/run.sh b/src/ci/run.sh index 960acc4de7d87..c161d9980369c 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -23,6 +23,7 @@ fi RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules" +RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps" # If we're deploying artifacts then we set the release channel, otherwise if # we're not deploying then we want to be sure to enable all assertions becauase @@ -47,13 +48,6 @@ else fi fi -# We want to enable usage of the `src/vendor` dir as much as possible, but not -# all test suites have all their deps in there (just the main bootstrap) so we -# have the ability to disable this flag -if [ "$NO_VENDOR" = "" ]; then - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-vendor" -fi - set -ex $SRC/configure $RUST_CONFIGURE_ARGS diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index 9962c6ec9af12..1bb7919c1d35b 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -42,6 +42,8 @@ fn main() { let path = env::args_os().skip(1).next().expect("need an argument"); let path = PathBuf::from(path); + let args: Vec = env::args().skip(1).collect(); + let mut bad = false; bins::check(&path, &mut bad); style::check(&path, &mut bad); @@ -49,7 +51,9 @@ fn main() { cargo::check(&path, &mut bad); features::check(&path, &mut bad); pal::check(&path, &mut bad); - deps::check(&path, &mut bad); + if !args.iter().any(|s| *s == "--no-vendor") { + deps::check(&path, &mut bad); + } if bad { panic!("some tidy checks failed");