Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
travis: Start uploading artifacts on commits
This commit starts adding the infrastructure for uploading release artifacts
from AppVeyor/Travis on each commit. The idea is that eventually we'll upload a
full release to AppVeyor/Travis in accordance with plans [outlined earlier].

Right now this configures Travis/Appveyor to upload all tarballs in the `dist`
directory, and various images are updated to actually produce tarballs in these
directories. These are nowhere near ready to be actual release artifacts, but
this should allow us to play around with it and test it out. Once this commit
lands we should start seeing artifacts uploaded on each commit.

[outlined earlier]: https://internals.rust-lang.org/t/rust-ci-release-infrastructure-changes/4489
  • Loading branch information
alexcrichton committed Jan 12, 2017
1 parent e357178 commit 3187672
Show file tree
Hide file tree
Showing 19 changed files with 148 additions and 64 deletions.
46 changes: 37 additions & 9 deletions .travis.yml
Expand Up @@ -13,20 +13,20 @@ osx_image: xcode8.2
matrix:
include:
# Linux builders, all docker images
- env: IMAGE=arm-android
- env: IMAGE=cross
- env: IMAGE=dist-arm-unknown-linux-gnueabi
- env: IMAGE=dist-x86_64-unknown-freebsd
- env: IMAGE=i686-gnu
- env: IMAGE=arm-android DEPLOY=1
- env: IMAGE=cross DEPLOY=1
- env: IMAGE=dist-arm-unknown-linux-gnueabi DEPLOY=1
- env: IMAGE=dist-x86_64-unknown-freebsd DEPLOY=1
- env: IMAGE=i686-gnu DEPLOY=1
- env: IMAGE=i686-gnu-nopt
- env: IMAGE=x86_64-gnu
- env: IMAGE=x86_64-gnu DEPLOY=1
- env: IMAGE=x86_64-gnu-full-bootstrap
- env: IMAGE=x86_64-gnu-aux
- env: IMAGE=x86_64-gnu-debug
- env: IMAGE=x86_64-gnu-nopt
- env: IMAGE=x86_64-gnu-make
- env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1
- env: IMAGE=x86_64-musl
- env: IMAGE=x86_64-musl DEPLOY=1
- env: IMAGE=x86_64-gnu-distcheck

# OSX builders
Expand All @@ -39,9 +39,10 @@ matrix:
curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
tar xJf - -C /usr/local/bin --strip-components=1
- env: >
RUST_CHECK_TARGET=check
SCRIPT="./x.py test && ./x.py dist"
RUST_CONFIGURE_ARGS=--build=i686-apple-darwin
SRC=.
DEPLOY=1
os: osx
install: *osx_install_sccache
- env: >
Expand All @@ -51,9 +52,10 @@ matrix:
os: osx
install: *osx_install_sccache
- env: >
RUST_CHECK_TARGET=
RUST_CHECK_TARGET=dist
RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
SRC=.
DEPLOY=1
os: osx
install: *osx_install_sccache
Expand Down Expand Up @@ -91,3 +93,29 @@ notifications:
cache:
directories:
- $HOME/docker

before_deploy:
- mkdir -p deploy/$TRAVIS_COMMIT
- >
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
cp build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'shasum -a 256 -b "{}" > "{}.sha256"' \;;
else
cp obj/build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'sha256sum -b "{}" > "{}.sha256"' \;;
fi
deploy:
- provider: s3
bucket: rust-lang-ci
skip_cleanup: true
local_dir: deploy
upload_dir: rustc-builds
acl: public_read
region: us-east-1
access_key_id: AKIAIPQVNYF2T3DTYIWQ
secret_access_key:
secure: "FBqDqOTeIPMu6v/WYPf4CFSlh9rLRZGKVtpLa5KkyuOhXRTrnEzBduEtS8/FMIxdQImvurhSvxWvqRybMOi4qoVfjMqqpHAI7uBbidbrvAcJoHNsx6BgUNVCIoH6a0UsAjTUtm6/YPIpzbHoLZXPL0GrHPMk6Mu04qVSmcYNWn4="
on:
branch: auto
condition: $DEPLOY = 1
38 changes: 34 additions & 4 deletions appveyor.yml
Expand Up @@ -8,10 +8,12 @@ environment:
# 32/64 bit MSVC
- MSYS_BITS: 64
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
DEPLOY: 1
- MSYS_BITS: 32
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
DEPLOY: 1

# MSVC makefiles
- MSYS_BITS: 64
Expand Down Expand Up @@ -50,10 +52,11 @@ environment:
# too long on appveyor and this is tested by rustbuild below.
- MSYS_BITS: 32
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
MINGW_DIR: mingw32
DEPLOY: 1

- MSYS_BITS: 32
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --disable-rustbuild
Expand All @@ -63,11 +66,12 @@ environment:
MINGW_DIR: mingw32

- MSYS_BITS: 64
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
MINGW_DIR: mingw64
DEPLOY: 1

clone_depth: 1
build: false
Expand Down Expand Up @@ -123,6 +127,32 @@ branches:
only:
- auto

before_deploy:
- ps: |
New-Item -Path deploy -ItemType directory
Get-ChildItem -Path build\dist -Filter '*.tar.gz' | Move-Item -Destination deploy
Get-FileHash .\deploy\* | ForEach-Object {
[io.file]::WriteAllText($_.Path + ".sha256", $_.Hash.ToLower() + "`n")
}
Get-ChildItem -Path deploy | Foreach-Object {
Push-AppveyorArtifact $_.FullName -FileName ${env:APPVEYOR_REPO_COMMIT}/$_
}
deploy:
- provider: S3
skip_cleanup: true
access_key_id: AKIAIPQVNYF2T3DTYIWQ
secret_access_key:
secure: +11jsUNFTQ9dq5Ad1i2+PeUJaXluFJ0zIJAXESE1dFT3Kdjku4/eDdgyjgsB6GnV
bucket: rust-lang-ci
set_public: true
region: us-east-1
artifact: /.*\.(tar.gz|sha256)/
folder: rustc-builds
on:
branch: auto
DEPLOY: 1

# init:
# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
# on_finish:
Expand Down
3 changes: 3 additions & 0 deletions src/bootstrap/check.rs
Expand Up @@ -457,6 +457,8 @@ fn krate_android(build: &Build,

let output = output(Command::new("adb").arg("shell").arg(&program));
println!("{}", output);

t!(fs::create_dir_all(build.out.join("tmp")));
build.run(Command::new("adb")
.arg("pull")
.arg(&log)
Expand Down Expand Up @@ -516,6 +518,7 @@ pub fn android_copy_libs(build: &Build,
}

println!("Android copy libs to emulator ({})", target);
build.run(Command::new("adb").arg("wait-for-device"));
build.run(Command::new("adb").arg("remount"));
build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR]));
build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR]));
Expand Down
50 changes: 31 additions & 19 deletions src/bootstrap/compile.rs
Expand Up @@ -16,7 +16,6 @@
//! compiler. This module is also responsible for assembling the sysroot as it
//! goes along from the output of the previous stage.

use std::cmp;
use std::collections::HashMap;
use std::fs::{self, File};
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -59,7 +58,7 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) {
}

build.run(&mut cargo);
update_mtime(&libstd_stamp(build, &compiler, target));
update_mtime(build, &libstd_stamp(build, &compiler, target));
}

/// Link all libstd rlibs/dylibs into the sysroot location.
Expand Down Expand Up @@ -145,7 +144,7 @@ pub fn test(build: &Build, target: &str, compiler: &Compiler) {
cargo.arg("--manifest-path")
.arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
build.run(&mut cargo);
update_mtime(&libtest_stamp(build, compiler, target));
update_mtime(build, &libtest_stamp(build, compiler, target));
}

/// Same as `std_link`, only for libtest
Expand Down Expand Up @@ -390,26 +389,39 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
}

/// Updates the mtime of a stamp file if necessary, only changing it if it's
/// older than some other file in the same directory.
/// older than some other library file in the same directory.
///
/// We don't know what file Cargo is going to output (because there's a hash in
/// the file name) but we know where it's going to put it. We use this helper to
/// detect changes to that output file by looking at the modification time for
/// all files in a directory and updating the stamp if any are newer.
fn update_mtime(path: &Path) {
let mut max = None;
if let Ok(entries) = path.parent().unwrap().join("deps").read_dir() {
for entry in entries.map(|e| t!(e)) {
if t!(entry.file_type()).is_file() {
let meta = t!(entry.metadata());
let time = FileTime::from_last_modification_time(&meta);
max = cmp::max(max, Some(time));
}
}
}

if !max.is_none() && max <= Some(mtime(path)) {
return
///
/// Note that we only consider Rust libraries as that's what we're interested in
/// propagating changes from. Files like executables are tracked elsewhere.
fn update_mtime(build: &Build, path: &Path) {
let entries = match path.parent().unwrap().join("deps").read_dir() {
Ok(entries) => entries,
Err(_) => return,
};
let files = entries.map(|e| t!(e)).filter(|e| t!(e.file_type()).is_file());
let files = files.filter(|e| {
let filename = e.file_name();
let filename = filename.to_str().unwrap();
filename.ends_with(".rlib") ||
filename.ends_with(".lib") ||
is_dylib(&filename)
});
let max = files.max_by_key(|entry| {
let meta = t!(entry.metadata());
FileTime::from_last_modification_time(&meta)
});
let max = match max {
Some(max) => max,
None => return,
};

if mtime(&max.path()) > mtime(path) {
build.verbose(&format!("updating {:?} as {:?} changed", path, max.path()));
t!(File::create(path));
}
t!(File::create(path));
}
11 changes: 7 additions & 4 deletions src/bootstrap/dist.rs
Expand Up @@ -357,14 +357,12 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
pub fn rust_src(build: &Build) {
println!("Dist src");

let plain_name = format!("rustc-{}-src", package_vers(build));
let name = format!("rust-src-{}", package_vers(build));
let image = tmpdir(build).join(format!("{}-image", name));
let _ = fs::remove_dir_all(&image);

let dst = image.join("lib/rustlib/src");
let dst_src = dst.join("rust");
let plain_dst_src = dst.join(&plain_name);
t!(fs::create_dir_all(&dst_src));

// This is the set of root paths which will become part of the source package
Expand Down Expand Up @@ -444,7 +442,11 @@ pub fn rust_src(build: &Build) {
build.run(&mut cmd);

// Rename directory, so that root folder of tarball has the correct name
t!(fs::rename(&dst_src, &plain_dst_src));
let plain_name = format!("rustc-{}-src", package_vers(build));
let plain_dst_src = tmpdir(build).join(&plain_name);
let _ = fs::remove_dir_all(&plain_dst_src);
t!(fs::create_dir_all(&plain_dst_src));
cp_r(&dst_src, &plain_dst_src);

// Create the version file
write_file(&plain_dst_src.join("version"), build.version.as_bytes());
Expand All @@ -453,10 +455,11 @@ pub fn rust_src(build: &Build) {
let mut cmd = Command::new("tar");
cmd.arg("-czf").arg(sanitize_sh(&rust_src_location(build)))
.arg(&plain_name)
.current_dir(&dst);
.current_dir(tmpdir(build));
build.run(&mut cmd);

t!(fs::remove_dir_all(&image));
t!(fs::remove_dir_all(&plain_dst_src));
}

fn install(src: &Path, dstdir: &Path, perms: u32) {
Expand Down
2 changes: 2 additions & 0 deletions src/bootstrap/flags.rs
Expand Up @@ -67,6 +67,7 @@ pub enum Subcommand {
},
Clean,
Dist {
paths: Vec<PathBuf>,
install: bool,
},
}
Expand Down Expand Up @@ -249,6 +250,7 @@ To learn more about a subcommand, run `./x.py <command> -h`
opts.optflag("", "install", "run installer as well");
m = parse(&opts);
Subcommand::Dist {
paths: remaining_as_path(&m),
install: m.opt_present("install"),
}
}
Expand Down
6 changes: 3 additions & 3 deletions src/bootstrap/step.rs
Expand Up @@ -615,7 +615,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
.default(true)
.only_host_build(true)
.run(move |s| dist::analysis(build, &s.compiler(), s.target));
rules.dist("install", "src")
rules.dist("install", "path/to/nowhere")
.dep(|s| s.name("default:dist"))
.run(move |s| install::install(build, s.stage, s.target));

Expand Down Expand Up @@ -932,11 +932,11 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]),
Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]),
Subcommand::Dist { install } => {
Subcommand::Dist { ref paths, install } => {
if install {
return vec![self.sbuild.name("install")]
} else {
(Kind::Dist, &[][..])
(Kind::Dist, &paths[..])
}
}
Subcommand::Clean => panic!(),
Expand Down
7 changes: 6 additions & 1 deletion src/ci/docker/arm-android/Dockerfile
Expand Up @@ -48,4 +48,9 @@ ENV RUST_CONFIGURE_ARGS \
--armv7-linux-androideabi-ndk=/android/ndk-arm-9 \
--i686-linux-android-ndk=/android/ndk-x86-9 \
--aarch64-linux-android-ndk=/android/ndk-aarch64
ENV XPY_CHECK test --target arm-linux-androideabi

# Just a smoke test in dist to see if this works for now, we should expand this
# to all the targets above eventually.
ENV SCRIPT \
python2.7 ../x.py test --target arm-linux-androideabi && \
python2.7 ../x.py dist --target arm-linux-androideabi
1 change: 0 additions & 1 deletion src/ci/docker/arm-android/start-emulator.sh
Expand Up @@ -13,5 +13,4 @@ set -ex
ANDROID_EMULATOR_FORCE_32BIT=true \
nohup nohup emulator @arm-18 -no-window -partition-size 2047 \
0<&- &>/dev/null &
adb wait-for-device
exec "$@"
7 changes: 6 additions & 1 deletion src/ci/docker/cross/Dockerfile
Expand Up @@ -56,7 +56,12 @@ ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
ENV RUST_CONFIGURE_ARGS \
--target=$TARGETS \
--enable-rustbuild
ENV RUST_CHECK_TARGET ""

# Just a smoke test in dist to see if this works for now, we should expand this
# to all the targets above eventually.
ENV SCRIPT \
python2.7 ../x.py build && \
python2.7 ../x.py dist --target wasm32-unknown-emscripten

ENV AR_s390x_unknown_linux_gnu=s390x-linux-gnu-ar \
CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc \
Expand Down
8 changes: 4 additions & 4 deletions src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile
Expand Up @@ -24,7 +24,7 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]

ENV RUST_CONFIGURE_ARGS --host=arm-unknown-linux-gnueabi
ENV XPY_RUN \
dist \
--host arm-unknown-linux-gnueabi \
--target arm-unknown-linux-gnueabi
ENV SCRIPT \
python2.7 ../x.py dist \
--host arm-unknown-linux-gnueabi \
--target arm-unknown-linux-gnueabi
8 changes: 4 additions & 4 deletions src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile
Expand Up @@ -32,7 +32,7 @@ ENV \
CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++

ENV RUST_CONFIGURE_ARGS --host=x86_64-unknown-freebsd
ENV XPY_RUN \
dist \
--host x86_64-unknown-freebsd \
--target x86_64-unknown-freebsd
ENV SCRIPT \
python2.7 ../x.py dist \
--host x86_64-unknown-freebsd \
--target x86_64-unknown-freebsd
2 changes: 1 addition & 1 deletion src/ci/docker/i686-gnu/Dockerfile
Expand Up @@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]

ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
ENV RUST_CHECK_TARGET check
ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist

0 comments on commit 3187672

Please sign in to comment.