Add AppVeyor configuration to the repo

We hope to move to AppVeyor in the near future off of Buildbot + EC2. This adds
an `appveyor.yml` configuration file which is ready to run builds on the auto
branch. This is also accompanied with a few minor fixes to the build system and
such to accomodate AppVeyor.

The intention is that we're not switching over to AppVeyor entirely just yet,
but rather we'll watch the builds for a week or so. If everything checks out
then we'll start gating on AppVeyor instead of Buildbot!

appveyor.yml

@@ -0,0 +1,110 @@
+environment:
+  matrix:
+  # 32/64 bit MSVC
+  - MSYS_BITS: 64
+    TARGET: x86_64-pc-windows-msvc
+    CHECK: check
+    CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions
+  - MSYS_BITS: 32
+    TARGET: i686-pc-windows-msvc
+    CHECK: check
+    CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions
+
+  # MSVC rustbuild
+  - MSYS_BITS: 64
+    CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions
+    TARGET: x86_64-pc-windows-msvc
+    CHECK: check
+
+  # MSVC cargotest
+  - MSYS_BITS: 64
+    CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions
+    TARGET: x86_64-pc-windows-msvc
+    CHECK: check-cargotest
+
+  # 32/64-bit MinGW builds.
+  #
+  # The MinGW builds unfortunately have to both download a custom toolchain and
+  # avoid the one installed by AppVeyor by default. Interestingly, though, for
+  # different reasons!
+  #
+  # For 32-bit the installed gcc toolchain on AppVeyor uses the pthread
+  # threading model. This is unfortunately not what we want, and if we compile
+  # with it then there's lots of link errors in the standard library (undefined
+  # references to pthread symbols).
+  #
+  # For 64-bit the installed gcc toolchain is currently 5.3.0 which
+  # unfortunately segfaults on Windows with --enable-llvm-assertions (segfaults
+  # in LLVM). See rust-lang/rust#28445 for more information, but to work around
+  # this we go back in time to 4.9.2 specifically.
+  #
+  # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+  # bucket, but they cleraly didn't originate there! The downloads originally
+  # came from the mingw-w64 SourceForge download site. Unfortunately
+  # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+  #
+  # And as a final point of note, the 32-bit MinGW build using the makefiles do
+  # *not* use debug assertions and llvm assertions. This is because they take
+  # too long on appveyor and this is tested by rustbuild below.
+  - MSYS_BITS: 32
+    TARGET: i686-pc-windows-gnu
+    CHECK: check
+    MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
+    MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
+    MINGW_DIR: mingw32
+
+  - MSYS_BITS: 32
+    CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions
+    TARGET: i686-pc-windows-gnu
+    CHECK: check
+    MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
+    MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
+    MINGW_DIR: mingw32
+
+  - MSYS_BITS: 64
+    CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions
+    TARGET: x86_64-pc-windows-gnu
+    CHECK: check
+    MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
+    MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
+    MINGW_DIR: mingw64
+
+clone_depth: 1
+build: false
+
+install:
+  # If we need to download a custom MinGW, do so here and set the path
+  # appropriately.
+  #
+  # Note that this *also* means that we're not using what is typically
+  # /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where
+  # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
+  # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
+  # file exists in there (which it doesn't by default).
+  - if defined MINGW_URL appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
+  - if defined MINGW_URL 7z x -y %MINGW_ARCHIVE% > nul
+  - if defined MINGW_URL set PATH=C:\Python27;%CD%\%MINGW_DIR%\bin;C:\msys64\usr\bin;%PATH%
+  - if defined MINGW_URL copy C:\Python27\python.exe C:\Python27\python2.7.exe
+
+  # Otherwise pull in the MinGW installed on appveyor
+  - if NOT defined MINGW_URL set PATH=C:\msys64\mingw%MSYS_BITS%\bin;C:\msys64\usr\bin;%PATH%
+
+test_script:
+  - sh ./configure
+          %CONFIGURE_ARGS%
+          --build=%TARGET%
+  - bash -c "make -j$(nproc)"
+  - bash -c "make %CHECK% -j$(nproc)"
+
+cache:
+  - build/%TARGET%/llvm -> src/rustllvm/llvm-auto-clean-trigger
+  - "%TARGET%/llvm -> src/rustllvm/llvm-auto-clean-trigger"
+
+branches:
+  only:
+    - auto
+
+# init:
+#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+# on_finish:
+#   - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

mk/llvm.mk

@@ -36,22 +36,27 @@ endif
 # If CFG_LLVM_ROOT is defined then we don't build LLVM ourselves
 ifeq ($(CFG_LLVM_ROOT),)
 
-LLVM_STAMP_$(1) = $$(CFG_LLVM_BUILD_DIR_$(1))/llvm-auto-clean-stamp
+LLVM_STAMP_$(1) = $(S)src/rustllvm/llvm-auto-clean-trigger
 LLVM_DONE_$(1) = $$(CFG_LLVM_BUILD_DIR_$(1))/llvm-finished-building
 
 $$(LLVM_CONFIG_$(1)): $$(LLVM_DONE_$(1))
 
-$$(LLVM_DONE_$(1)): $$(LLVM_DEPS_TARGET_$(1)) $$(LLVM_STAMP_$(1))
-	@$$(call E, cmake: llvm)
 ifneq ($$(CFG_NINJA),)
-	$$(Q)$$(CFG_NINJA) -C $$(CFG_LLVM_BUILD_DIR_$(1))
+BUILD_LLVM_$(1) := $$(CFG_NINJA) -C $$(CFG_LLVM_BUILD_DIR_$(1))
 else ifeq ($$(findstring msvc,$(1)),msvc)
-	$$(Q)$$(CFG_CMAKE) --build $$(CFG_LLVM_BUILD_DIR_$(1)) \
-		--config $$(LLVM_BUILD_CONFIG_MODE)
+BUILD_LLVM_$(1) := $$(CFG_CMAKE) --build $$(CFG_LLVM_BUILD_DIR_$(1)) \
+			--config $$(LLVM_BUILD_CONFIG_MODE)
 else
-	$$(Q)$$(MAKE) -C $$(CFG_LLVM_BUILD_DIR_$(1))
+BUILD_LLVM_$(1) := $$(MAKE) -C $$(CFG_LLVM_BUILD_DIR_$(1))
 endif
-	$$(Q)touch $$@
+
+$$(LLVM_DONE_$(1)): $$(LLVM_DEPS_TARGET_$(1)) $$(LLVM_STAMP_$(1))
+	@$$(call E, cmake: llvm)
+	$$(Q)if ! cmp $$(LLVM_STAMP_$(1)) $$(LLVM_DONE_$(1)); then \
+		$$(MAKE) clean-llvm$(1); \
+		$$(BUILD_LLVM_$(1)); \
+	fi
+	$$(Q)cp $$(LLVM_STAMP_$(1)) $$@
 
 ifneq ($$(CFG_NINJA),)
 clean-llvm$(1):
@@ -75,17 +80,6 @@ endif
 
 $$(LLVM_AR_$(1)): $$(LLVM_CONFIG_$(1))
 
-# This is used to independently force an LLVM clean rebuild
-# when we changed something not otherwise captured by builtin
-# dependencies. In these cases, commit a change that touches
-# the stamp in the source dir.
-$$(LLVM_STAMP_$(1)): $$(S)src/rustllvm/llvm-auto-clean-trigger
-	@$$(call E, make: cleaning llvm)
-	$$(Q)touch $$@.start_time
-	$$(Q)$$(MAKE) clean-llvm$(1)
-	@$$(call E, make: done cleaning llvm)
-	touch -r $$@.start_time $$@ && rm $$@.start_time
-
 ifeq ($$(CFG_ENABLE_LLVM_STATIC_STDCPP),1)
 LLVM_STDCPP_RUSTFLAGS_$(1) = -L "$$(dir $$(shell $$(CC_$(1)) $$(CFG_GCCISH_CFLAGS_$(1)) \
 					-print-file-name=lib$(CFG_STDCPP_NAME).a))"

src/bootstrap/config.rs

@@ -120,6 +120,7 @@ struct Build {
     rustc: Option<String>,
     compiler_docs: Option<bool>,
     docs: Option<bool>,
+    submodules: Option<bool>,
 }
 
 /// TOML representation of how the LLVM build is configured.
@@ -225,6 +226,7 @@ impl Config {
         config.cargo = build.cargo.map(PathBuf::from);
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
+        set(&mut config.submodules, build.submodules);
 
         if let Some(ref llvm) = toml.llvm {
             set(&mut config.ccache, llvm.ccache);

src/bootstrap/config.toml.example

@@ -76,6 +76,9 @@
 # library and facade crates.
 #compiler-docs = false
 
+# Indicate whether submodules are managed and updated automatically.
+#submodules = true
+
 # =============================================================================
 # Options for compiling Rust code itself
 # =============================================================================

src/bootstrap/native.rs

@@ -18,9 +18,10 @@
 //! LLVM and compiler-rt are essentially just wired up to everything else to
 //! ensure that they're always in place if needed.
 
+use std::fs::{self, File};
+use std::io::{Read, Write};
 use std::path::Path;
 use std::process::Command;
-use std::fs::{self, File};
 
 use build_helper::output;
 use cmake;
@@ -43,11 +44,17 @@ pub fn llvm(build: &Build, target: &str) {
     // artifacts are missing) then we keep going, otherwise we bail out.
     let dst = build.llvm_out(target);
     let stamp = build.src.join("src/rustllvm/llvm-auto-clean-trigger");
+    let mut stamp_contents = String::new();
+    t!(t!(File::open(&stamp)).read_to_string(&mut stamp_contents));
     let done_stamp = dst.join("llvm-finished-building");
-    build.clear_if_dirty(&dst, &stamp);
-    if fs::metadata(&done_stamp).is_ok() {
-        return
+    if done_stamp.exists() {
+        let mut done_contents = String::new();
+        t!(t!(File::open(&done_stamp)).read_to_string(&mut done_contents));
+        if done_contents == stamp_contents {
+            return
+        }
     }
+    drop(fs::remove_dir_all(&dst));
 
     println!("Building LLVM for {}", target);
 
@@ -73,7 +80,9 @@ pub fn llvm(build: &Build, target: &str) {
        .define("WITH_POLLY", "OFF")
        .define("LLVM_ENABLE_TERMINFO", "OFF")
        .define("LLVM_ENABLE_LIBEDIT", "OFF")
-       .define("LLVM_PARALLEL_COMPILE_JOBS", build.jobs().to_string());
+       .define("LLVM_PARALLEL_COMPILE_JOBS", build.jobs().to_string())
+       .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
+       .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
 
     if target.starts_with("i686") {
         cfg.define("LLVM_BUILD_32_BITS", "ON");
@@ -86,9 +95,7 @@ pub fn llvm(build: &Build, target: &str) {
         //        actually exists most of the time in normal installs of LLVM.
         let host = build.llvm_out(&build.config.build).join("bin/llvm-tblgen");
         cfg.define("CMAKE_CROSSCOMPILING", "True")
-           .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
-           .define("LLVM_TABLEGEN", &host)
-           .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
+           .define("LLVM_TABLEGEN", &host);
     }
 
     // MSVC handles compiler business itself
@@ -114,7 +121,7 @@ pub fn llvm(build: &Build, target: &str) {
     //        tools and libs on all platforms.
     cfg.build();
 
-    t!(File::create(&done_stamp));
+    t!(t!(File::create(&done_stamp)).write_all(stamp_contents.as_bytes()));
 }
 
 fn check_llvm_version(build: &Build, llvm_config: &Path) {

src/rustllvm/llvm-auto-clean-trigger

@@ -1,4 +1,4 @@
 # If this file is modified, then llvm will be forcibly cleaned and then rebuilt.
 # The actual contents of this file do not matter, but to trigger a change on the
 # build bots then the contents should be changed so git updates the mtime.
-2016-10-10
+2016-10-10b

src/test/run-make/tools.mk

@@ -22,9 +22,6 @@ RLIB_GLOB = lib$(1)*.rlib
 BIN = $(1)
 
 UNAME = $(shell uname)
-ifneq (,$(findstring MINGW,$(UNAME)))
-IS_WINDOWS=1
-endif
 
 ifeq ($(UNAME),Darwin)
 RUN = $(TARGET_RPATH_ENV) $(RUN_BINFILE)

src/tools/cargotest/main.rs

@@ -36,6 +36,20 @@ const TEST_REPOS: &'static [Test] = &[Test {
 
 
 fn main() {
+    // One of the projects being tested here is Cargo, and when being tested
+    // Cargo will at some point call `nmake.exe` on Windows MSVC. Unfortunately
+    // `nmake` will read these two environment variables below and try to
+    // intepret them. We're likely being run, however, from MSYS `make` which
+    // uses the same variables.
+    //
+    // As a result, to prevent confusion and errors, we remove these variables
+    // from our environment to prevent passing MSYS make flags to nmake, causing
+    // it to blow up.
+    if cfg!(target_env = "msvc") {
+        env::remove_var("MAKE");
+        env::remove_var("MAKEFLAGS");
+    }
+
     let args = env::args().collect::<Vec<_>>();
     let ref cargo = args[1];
     let out_dir = Path::new(&args[2]);

src/tools/compiletest/src/runtest.rs

@@ -2105,12 +2105,17 @@ actual:\n\
                                                  .collect::<Vec<_>>().join(" ");
 
             cmd.env("IS_MSVC", "1")
+               .env("IS_WINDOWS", "1")
                .env("MSVC_LIB", format!("'{}' -nologo", lib.display()))
                .env("CC", format!("'{}' {}", self.config.cc, cflags))
                .env("CXX", &self.config.cxx);
         } else {
             cmd.env("CC", format!("{} {}", self.config.cc, self.config.cflags))
                .env("CXX", format!("{} {}", self.config.cxx, self.config.cflags));
+
+            if self.config.target.contains("windows") {
+                cmd.env("IS_WINDOWS", "1");
+            }
         }
 
         let output = cmd.output().expect("failed to spawn `make`");