diff --git a/.travis.yml b/.travis.yml index 2b5eaecb0b42c..505795d33e5ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,20 +13,20 @@ osx_image: xcode8.2 matrix: include: # Linux builders, all docker images - - env: IMAGE=arm-android - - env: IMAGE=cross - - env: IMAGE=dist-arm-unknown-linux-gnueabi - - env: IMAGE=dist-x86_64-unknown-freebsd - - env: IMAGE=i686-gnu + - env: IMAGE=arm-android DEPLOY=1 + - env: IMAGE=cross DEPLOY=1 + - env: IMAGE=dist-arm-unknown-linux-gnueabi DEPLOY=1 + - env: IMAGE=dist-x86_64-unknown-freebsd DEPLOY=1 + - env: IMAGE=i686-gnu DEPLOY=1 - env: IMAGE=i686-gnu-nopt - - env: IMAGE=x86_64-gnu + - env: IMAGE=x86_64-gnu DEPLOY=1 - env: IMAGE=x86_64-gnu-full-bootstrap - env: IMAGE=x86_64-gnu-aux - env: IMAGE=x86_64-gnu-debug - env: IMAGE=x86_64-gnu-nopt - env: IMAGE=x86_64-gnu-make - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1 - - env: IMAGE=x86_64-musl + - env: IMAGE=x86_64-musl DEPLOY=1 - env: IMAGE=x86_64-gnu-distcheck # OSX builders @@ -39,9 +39,10 @@ matrix: curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 | tar xJf - -C /usr/local/bin --strip-components=1 - env: > - RUST_CHECK_TARGET=check + SCRIPT="./x.py test && ./x.py dist" RUST_CONFIGURE_ARGS=--build=i686-apple-darwin SRC=. + DEPLOY=1 os: osx install: *osx_install_sccache - env: > @@ -51,9 +52,10 @@ matrix: os: osx install: *osx_install_sccache - env: > - RUST_CHECK_TARGET= + RUST_CHECK_TARGET=dist RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios SRC=. + DEPLOY=1 os: osx install: *osx_install_sccache @@ -91,3 +93,29 @@ notifications: cache: directories: - $HOME/docker + +before_deploy: + - mkdir -p deploy/$TRAVIS_COMMIT + - > + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + cp build/dist/*.tar.gz deploy/$TRAVIS_COMMIT && + find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'shasum -a 256 -b "{}" > "{}.sha256"' \;; + else + cp obj/build/dist/*.tar.gz deploy/$TRAVIS_COMMIT && + find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'sha256sum -b "{}" > "{}.sha256"' \;; + fi + +deploy: + - provider: s3 + bucket: rust-lang-ci + skip_cleanup: true + local_dir: deploy + upload_dir: rustc-builds + acl: public_read + region: us-east-1 + access_key_id: AKIAIPQVNYF2T3DTYIWQ + secret_access_key: + secure: "FBqDqOTeIPMu6v/WYPf4CFSlh9rLRZGKVtpLa5KkyuOhXRTrnEzBduEtS8/FMIxdQImvurhSvxWvqRybMOi4qoVfjMqqpHAI7uBbidbrvAcJoHNsx6BgUNVCIoH6a0UsAjTUtm6/YPIpzbHoLZXPL0GrHPMk6Mu04qVSmcYNWn4=" + on: + branch: auto + condition: $DEPLOY = 1 diff --git a/appveyor.yml b/appveyor.yml index a4b5cbc115253..346ef0d8faad7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,10 +8,12 @@ environment: # 32/64 bit MSVC - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - RUST_CHECK_TARGET: check + SCRIPT: python x.py test && python x.py dist + DEPLOY: 1 - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - RUST_CHECK_TARGET: check + SCRIPT: python x.py test && python x.py dist + DEPLOY: 1 # MSVC makefiles - MSYS_BITS: 64 @@ -50,10 +52,11 @@ environment: # too long on appveyor and this is tested by rustbuild below. - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - RUST_CHECK_TARGET: check + SCRIPT: python x.py test && python x.py dist MINGW_URL: https://s3.amazonaws.com/rust-lang-ci MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z MINGW_DIR: mingw32 + DEPLOY: 1 - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --disable-rustbuild @@ -63,11 +66,12 @@ environment: MINGW_DIR: mingw32 - MSYS_BITS: 64 - RUST_CHECK_TARGET: check + SCRIPT: python x.py test && python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu MINGW_URL: https://s3.amazonaws.com/rust-lang-ci MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z MINGW_DIR: mingw64 + DEPLOY: 1 clone_depth: 1 build: false @@ -123,6 +127,32 @@ branches: only: - auto +before_deploy: + - ps: | + New-Item -Path deploy -ItemType directory + Get-ChildItem -Path build\dist -Filter '*.tar.gz' | Move-Item -Destination deploy + Get-FileHash .\deploy\* | ForEach-Object { + [io.file]::WriteAllText($_.Path + ".sha256", $_.Hash.ToLower() + "`n") + } + Get-ChildItem -Path deploy | Foreach-Object { + Push-AppveyorArtifact $_.FullName -FileName ${env:APPVEYOR_REPO_COMMIT}/$_ + } + +deploy: + - provider: S3 + skip_cleanup: true + access_key_id: AKIAIPQVNYF2T3DTYIWQ + secret_access_key: + secure: +11jsUNFTQ9dq5Ad1i2+PeUJaXluFJ0zIJAXESE1dFT3Kdjku4/eDdgyjgsB6GnV + bucket: rust-lang-ci + set_public: true + region: us-east-1 + artifact: /.*\.(tar.gz|sha256)/ + folder: rustc-builds + on: + branch: auto + DEPLOY: 1 + # init: # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) # on_finish: diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index f2fddf6e2ef3a..cafdd3aec65bd 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -457,6 +457,8 @@ fn krate_android(build: &Build, let output = output(Command::new("adb").arg("shell").arg(&program)); println!("{}", output); + + t!(fs::create_dir_all(build.out.join("tmp"))); build.run(Command::new("adb") .arg("pull") .arg(&log) @@ -516,6 +518,7 @@ pub fn android_copy_libs(build: &Build, } println!("Android copy libs to emulator ({})", target); + build.run(Command::new("adb").arg("wait-for-device")); build.run(Command::new("adb").arg("remount")); build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR])); build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR])); diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 98793d5a2868b..0eeb799672cf2 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -16,7 +16,6 @@ //! compiler. This module is also responsible for assembling the sysroot as it //! goes along from the output of the previous stage. -use std::cmp; use std::collections::HashMap; use std::fs::{self, File}; use std::path::{Path, PathBuf}; @@ -59,7 +58,7 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) { } build.run(&mut cargo); - update_mtime(&libstd_stamp(build, &compiler, target)); + update_mtime(build, &libstd_stamp(build, &compiler, target)); } /// Link all libstd rlibs/dylibs into the sysroot location. @@ -145,7 +144,7 @@ pub fn test(build: &Build, target: &str, compiler: &Compiler) { cargo.arg("--manifest-path") .arg(build.src.join("src/rustc/test_shim/Cargo.toml")); build.run(&mut cargo); - update_mtime(&libtest_stamp(build, compiler, target)); + update_mtime(build, &libtest_stamp(build, compiler, target)); } /// Same as `std_link`, only for libtest @@ -390,26 +389,39 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) { } /// Updates the mtime of a stamp file if necessary, only changing it if it's -/// older than some other file in the same directory. +/// older than some other library file in the same directory. /// /// We don't know what file Cargo is going to output (because there's a hash in /// the file name) but we know where it's going to put it. We use this helper to /// detect changes to that output file by looking at the modification time for /// all files in a directory and updating the stamp if any are newer. -fn update_mtime(path: &Path) { - let mut max = None; - if let Ok(entries) = path.parent().unwrap().join("deps").read_dir() { - for entry in entries.map(|e| t!(e)) { - if t!(entry.file_type()).is_file() { - let meta = t!(entry.metadata()); - let time = FileTime::from_last_modification_time(&meta); - max = cmp::max(max, Some(time)); - } - } - } - - if !max.is_none() && max <= Some(mtime(path)) { - return +/// +/// Note that we only consider Rust libraries as that's what we're interested in +/// propagating changes from. Files like executables are tracked elsewhere. +fn update_mtime(build: &Build, path: &Path) { + let entries = match path.parent().unwrap().join("deps").read_dir() { + Ok(entries) => entries, + Err(_) => return, + }; + let files = entries.map(|e| t!(e)).filter(|e| t!(e.file_type()).is_file()); + let files = files.filter(|e| { + let filename = e.file_name(); + let filename = filename.to_str().unwrap(); + filename.ends_with(".rlib") || + filename.ends_with(".lib") || + is_dylib(&filename) + }); + let max = files.max_by_key(|entry| { + let meta = t!(entry.metadata()); + FileTime::from_last_modification_time(&meta) + }); + let max = match max { + Some(max) => max, + None => return, + }; + + if mtime(&max.path()) > mtime(path) { + build.verbose(&format!("updating {:?} as {:?} changed", path, max.path())); + t!(File::create(path)); } - t!(File::create(path)); } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index ad851e448ea7c..dc45d3817fe30 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -357,14 +357,12 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) { pub fn rust_src(build: &Build) { println!("Dist src"); - let plain_name = format!("rustc-{}-src", package_vers(build)); let name = format!("rust-src-{}", package_vers(build)); let image = tmpdir(build).join(format!("{}-image", name)); let _ = fs::remove_dir_all(&image); let dst = image.join("lib/rustlib/src"); let dst_src = dst.join("rust"); - let plain_dst_src = dst.join(&plain_name); t!(fs::create_dir_all(&dst_src)); // This is the set of root paths which will become part of the source package @@ -444,7 +442,11 @@ pub fn rust_src(build: &Build) { build.run(&mut cmd); // Rename directory, so that root folder of tarball has the correct name - t!(fs::rename(&dst_src, &plain_dst_src)); + let plain_name = format!("rustc-{}-src", package_vers(build)); + let plain_dst_src = tmpdir(build).join(&plain_name); + let _ = fs::remove_dir_all(&plain_dst_src); + t!(fs::create_dir_all(&plain_dst_src)); + cp_r(&dst_src, &plain_dst_src); // Create the version file write_file(&plain_dst_src.join("version"), build.version.as_bytes()); @@ -453,10 +455,11 @@ pub fn rust_src(build: &Build) { let mut cmd = Command::new("tar"); cmd.arg("-czf").arg(sanitize_sh(&rust_src_location(build))) .arg(&plain_name) - .current_dir(&dst); + .current_dir(tmpdir(build)); build.run(&mut cmd); t!(fs::remove_dir_all(&image)); + t!(fs::remove_dir_all(&plain_dst_src)); } fn install(src: &Path, dstdir: &Path, perms: u32) { diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index b2412fbb3c844..1be823417d765 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -67,6 +67,7 @@ pub enum Subcommand { }, Clean, Dist { + paths: Vec, install: bool, }, } @@ -249,6 +250,7 @@ To learn more about a subcommand, run `./x.py -h` opts.optflag("", "install", "run installer as well"); m = parse(&opts); Subcommand::Dist { + paths: remaining_as_path(&m), install: m.opt_present("install"), } } diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index a8a047a29285b..65f3d241b491b 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -615,7 +615,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .default(true) .only_host_build(true) .run(move |s| dist::analysis(build, &s.compiler(), s.target)); - rules.dist("install", "src") + rules.dist("install", "path/to/nowhere") .dep(|s| s.name("default:dist")) .run(move |s| install::install(build, s.stage, s.target)); @@ -932,11 +932,11 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]), Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]), Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]), - Subcommand::Dist { install } => { + Subcommand::Dist { ref paths, install } => { if install { return vec![self.sbuild.name("install")] } else { - (Kind::Dist, &[][..]) + (Kind::Dist, &paths[..]) } } Subcommand::Clean => panic!(), diff --git a/src/ci/docker/arm-android/Dockerfile b/src/ci/docker/arm-android/Dockerfile index a40e76839eccd..e2ef8070b51af 100644 --- a/src/ci/docker/arm-android/Dockerfile +++ b/src/ci/docker/arm-android/Dockerfile @@ -48,4 +48,9 @@ ENV RUST_CONFIGURE_ARGS \ --armv7-linux-androideabi-ndk=/android/ndk-arm-9 \ --i686-linux-android-ndk=/android/ndk-x86-9 \ --aarch64-linux-android-ndk=/android/ndk-aarch64 -ENV XPY_CHECK test --target arm-linux-androideabi + +# Just a smoke test in dist to see if this works for now, we should expand this +# to all the targets above eventually. +ENV SCRIPT \ + python2.7 ../x.py test --target arm-linux-androideabi && \ + python2.7 ../x.py dist --target arm-linux-androideabi diff --git a/src/ci/docker/arm-android/start-emulator.sh b/src/ci/docker/arm-android/start-emulator.sh index fcf42c1819ee7..24c477d87f1a7 100755 --- a/src/ci/docker/arm-android/start-emulator.sh +++ b/src/ci/docker/arm-android/start-emulator.sh @@ -13,5 +13,4 @@ set -ex ANDROID_EMULATOR_FORCE_32BIT=true \ nohup nohup emulator @arm-18 -no-window -partition-size 2047 \ 0<&- &>/dev/null & -adb wait-for-device exec "$@" diff --git a/src/ci/docker/cross/Dockerfile b/src/ci/docker/cross/Dockerfile index c5aa323f50cd2..6a6cc23698be2 100644 --- a/src/ci/docker/cross/Dockerfile +++ b/src/ci/docker/cross/Dockerfile @@ -56,7 +56,12 @@ ENV TARGETS=$TARGETS,wasm32-unknown-emscripten ENV RUST_CONFIGURE_ARGS \ --target=$TARGETS \ --enable-rustbuild -ENV RUST_CHECK_TARGET "" + +# Just a smoke test in dist to see if this works for now, we should expand this +# to all the targets above eventually. +ENV SCRIPT \ + python2.7 ../x.py build && \ + python2.7 ../x.py dist --target wasm32-unknown-emscripten ENV AR_s390x_unknown_linux_gnu=s390x-linux-gnu-ar \ CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc \ diff --git a/src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile b/src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile index 9b0f1b7a0a763..841464a42700d 100644 --- a/src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile +++ b/src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile @@ -24,7 +24,7 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS --host=arm-unknown-linux-gnueabi -ENV XPY_RUN \ - dist \ - --host arm-unknown-linux-gnueabi \ - --target arm-unknown-linux-gnueabi +ENV SCRIPT \ + python2.7 ../x.py dist \ + --host arm-unknown-linux-gnueabi \ + --target arm-unknown-linux-gnueabi diff --git a/src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile b/src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile index f1a6ccf9ebcf4..44cc57c2662d6 100644 --- a/src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile +++ b/src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile @@ -32,7 +32,7 @@ ENV \ CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++ ENV RUST_CONFIGURE_ARGS --host=x86_64-unknown-freebsd -ENV XPY_RUN \ - dist \ - --host x86_64-unknown-freebsd \ - --target x86_64-unknown-freebsd +ENV SCRIPT \ + python2.7 ../x.py dist \ + --host x86_64-unknown-freebsd \ + --target x86_64-unknown-freebsd diff --git a/src/ci/docker/i686-gnu/Dockerfile b/src/ci/docker/i686-gnu/Dockerfile index e4310232d78cf..6583f09be368c 100644 --- a/src/ci/docker/i686-gnu/Dockerfile +++ b/src/ci/docker/i686-gnu/Dockerfile @@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu -ENV RUST_CHECK_TARGET check +ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist diff --git a/src/ci/docker/x86_64-gnu-debug/Dockerfile b/src/ci/docker/x86_64-gnu-debug/Dockerfile index 9ec8c6059ec31..f54e4e562f889 100644 --- a/src/ci/docker/x86_64-gnu-debug/Dockerfile +++ b/src/ci/docker/x86_64-gnu-debug/Dockerfile @@ -26,4 +26,4 @@ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --enable-debug \ --enable-optimize -ENV RUST_CHECK_TARGET "" +ENV SCRIPT python2.7 ../x.py build diff --git a/src/ci/docker/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/x86_64-gnu-distcheck/Dockerfile index 880c70866b0f4..57a2c103f1e7d 100644 --- a/src/ci/docker/x86_64-gnu-distcheck/Dockerfile +++ b/src/ci/docker/x86_64-gnu-distcheck/Dockerfile @@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu -ENV XPY_RUN test distcheck +ENV SCRIPT python2.7 ../x.py test distcheck diff --git a/src/ci/docker/x86_64-gnu-full-bootstrap/Dockerfile b/src/ci/docker/x86_64-gnu-full-bootstrap/Dockerfile index 7c079e45751c1..206103b92e94d 100644 --- a/src/ci/docker/x86_64-gnu-full-bootstrap/Dockerfile +++ b/src/ci/docker/x86_64-gnu-full-bootstrap/Dockerfile @@ -25,4 +25,4 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --enable-full-bootstrap -ENV RUST_CHECK_TARGET "" +ENV SCRIPT python2.7 ../x.py build diff --git a/src/ci/docker/x86_64-gnu/Dockerfile b/src/ci/docker/x86_64-gnu/Dockerfile index 9d8b75c80c7f2..6919487e17c3e 100644 --- a/src/ci/docker/x86_64-gnu/Dockerfile +++ b/src/ci/docker/x86_64-gnu/Dockerfile @@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu -ENV RUST_CHECK_TARGET check +ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist diff --git a/src/ci/docker/x86_64-musl/Dockerfile b/src/ci/docker/x86_64-musl/Dockerfile index 49954157ec023..11b85a6c8fdfb 100644 --- a/src/ci/docker/x86_64-musl/Dockerfile +++ b/src/ci/docker/x86_64-musl/Dockerfile @@ -29,6 +29,7 @@ RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | ENV RUST_CONFIGURE_ARGS \ --target=x86_64-unknown-linux-musl \ --musl-root-x86_64=/musl-x86_64 -ENV RUST_CHECK_TARGET check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu ENV PATH=$PATH:/musl-x86_64/bin -ENV XPY_CHECK test --target x86_64-unknown-linux-musl +ENV SCRIPT \ + python2.7 ../x.py test --target x86_64-unknown-linux-musl && \ + python2.7 ../x.py dist --target x86_64-unknown-linux-musl diff --git a/src/ci/run.sh b/src/ci/run.sh index 3dc3cf9150193..737c3622c694e 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -43,14 +43,10 @@ else ncpus=$(nproc) fi -if [ ! -z "$XPY_RUN" ]; then - exec python2.7 $SRC/x.py $XPY_RUN +if [ ! -z "$SCRIPT" ]; then + sh -x -c "$SCRIPT" else make -j $ncpus tidy make -j $ncpus - if [ ! -z "$XPY_CHECK" ]; then - exec python2.7 $SRC/x.py $XPY_CHECK - else - exec make $RUST_CHECK_TARGET -j $ncpus - fi + make $RUST_CHECK_TARGET -j $ncpus fi