significantly increase test coverage

.gitattributes

@@ -1,4 +1,5 @@
 *.zig text eol=lf
+*.txt text eol=lf
 langref.html.in text eol=lf
 
 deps/* linguist-vendored

build.zig

@@ -70,6 +70,7 @@ pub fn build(b: *Builder) !void {
     const skip_release_fast = b.option(bool, "skip-release-fast", "Main test suite skips release-fast builds") orelse skip_release;
     const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release;
     const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false;
+    const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
     const skip_self_hosted = b.option(bool, "skip-self-hosted", "Main test suite skips building self hosted compiler") orelse false;
     if (!skip_self_hosted) {
         // TODO re-enable this after https://github.com/ziglang/zig/issues/2377
@@ -96,6 +97,10 @@ pub fn build(b: *Builder) !void {
 
     const test_filter = b.option([]const u8, "test-filter", "Skip tests that do not match filter");
 
+    const is_wine_enabled = b.option(bool, "enable-wine", "Use Wine to run cross compiled Windows tests") orelse false;
+    const is_qemu_enabled = b.option(bool, "enable-qemu", "Use QEMU to run cross compiled foreign architecture tests") orelse false;
+    const glibc_multi_dir = b.option([]const u8, "enable-foreign-glibc", "Provide directory with glibc installations to run cross compiled tests that link glibc");
+
     const test_stage2_step = b.step("test-stage2", "Run the stage2 compiler tests");
     test_stage2_step.dependOn(&test_stage2.step);
 
@@ -122,19 +127,16 @@ pub fn build(b: *Builder) !void {
     }
     const modes = chosen_modes[0..chosen_mode_index];
 
-    const multi_and_single = [_]bool{ false, true };
-    const just_multi = [_]bool{false};
-
     // run stage1 `zig fmt` on this build.zig file just to make sure it works
     test_step.dependOn(&fmt_build_zig.step);
     const fmt_step = b.step("test-fmt", "Run zig fmt against build.zig to make sure it works");
     fmt_step.dependOn(&fmt_build_zig.step);
 
-    test_step.dependOn(tests.addPkgTests(b, test_filter, "test/stage1/behavior.zig", "behavior", "Run the behavior tests", modes, multi_and_single, skip_non_native));
+    test_step.dependOn(tests.addPkgTests(b, test_filter, "test/stage1/behavior.zig", "behavior", "Run the behavior tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, glibc_multi_dir));
 
-    test_step.dependOn(tests.addPkgTests(b, test_filter, "std/std.zig", "std", "Run the standard library tests", modes, multi_and_single, skip_non_native));
+    test_step.dependOn(tests.addPkgTests(b, test_filter, "std/std.zig", "std", "Run the standard library tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, glibc_multi_dir));
 
-    test_step.dependOn(tests.addPkgTests(b, test_filter, "std/special/compiler_rt.zig", "compiler-rt", "Run the compiler_rt tests", modes, just_multi, skip_non_native));
+    test_step.dependOn(tests.addPkgTests(b, test_filter, "std/special/compiler_rt.zig", "compiler-rt", "Run the compiler_rt tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, glibc_multi_dir));
 
     test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
     test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes));

ci/azure/linux_script

@@ -12,15 +12,15 @@ sudo apt-get update -q
 
 sudo apt-get remove -y llvm-*
 sudo rm -rf /usr/local/*
-sudo apt-get install -y libxml2-dev libclang-9-dev llvm-9 llvm-9-dev cmake s3cmd gcc-7 g++-7
+sudo apt-get install -y libxml2-dev libclang-9-dev llvm-9 llvm-9-dev cmake s3cmd gcc-7 g++-7 qemu
 
 export CC=gcc-7
 export CXX=g++-7
 mkdir build
 cd build
 cmake .. -DCMAKE_BUILD_TYPE=Release
 make -j2 install
-./zig build --build-file ../build.zig test
+./zig build test -Denable-qemu
 
 if [ "${BUILD_REASON}" != "PullRequest" ]; then
   ARTIFACTSDIR="$BUILDDIR/artifacts"

ci/azure/macos_script

@@ -70,7 +70,7 @@ mkdir build
 cd build
 cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$(pwd)/release -DZIG_STATIC=ON
 make $JOBS install
-release/bin/zig build --build-file ../build.zig test
+release/bin/zig build test
 
 if [ "${BUILD_REASON}" != "PullRequest" ]; then
   mv ../LICENSE release/

ci/azure/pipelines.yml

@@ -27,7 +27,7 @@ jobs:
     displayName: 'Build and test'
 - job: BuildWindows
   pool:
-    vmImage: 'vs2017-win2016'
+    vmImage: 'windows-2019'
 
   timeoutInMinutes: 360
 

ci/azure/windows_script.bat

@@ -10,17 +10,18 @@ SET "PATH=%PREVPATH%"
 SET "MSYSTEM=%PREVMSYSTEM%"
 
 SET "ZIGBUILDDIR=%SRCROOT%\build"
-SET "ZIGINSTALLDIR=%ZIGBUILDDIR%\release"
+SET "ZIGINSTALLDIR=%ZIGBUILDDIR%\dist"
 SET "ZIGPREFIXPATH=%SRCROOT%\llvm+clang-9.0.0-win64-msvc-release"
 
-call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
 
 mkdir %ZIGBUILDDIR%
 cd %ZIGBUILDDIR%
-cmake.exe .. -Thost=x64 -G"Visual Studio 15 2017 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release || exit /b
-msbuild /p:Configuration=Release INSTALL.vcxproj || exit /b
+REM Here we use MinSizeRel instead of Release to work around https://github.com/ziglang/zig/issues/3024
+cmake.exe .. -Thost=x64 -G"Visual Studio 16 2019" -A x64 "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=MinSizeRel || exit /b
+msbuild /p:Configuration=MinSizeRel INSTALL.vcxproj || exit /b
 
-"%ZIGINSTALLDIR%\bin\zig.exe" build --build-file ..\build.zig test || exit /b
+"%ZIGINSTALLDIR%\bin\zig.exe" build test || exit /b
 
 set "PATH=%CD:~0,2%\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem"
 SET "MSYSTEM=MINGW64"

ci/azure/windows_upload

@@ -6,16 +6,15 @@ set -e
 if [ "${BUILD_REASON}" != "PullRequest" ]; then
   cd "$ZIGBUILDDIR"
 
-  rm release/*.exe
-  mv ../LICENSE release/
-  mv ../zig-cache/langref.html release/
-  mv release/bin/zig.exe release/
-  rmdir release/bin
+  mv ../LICENSE dist/
+  mv ../zig-cache/langref.html dist/
+  mv dist/bin/zig.exe dist/
+  rmdir dist/bin
 
-  VERSION=$(release/zig.exe version)
+  VERSION=$(dist/zig.exe version)
   DIRNAME="zig-windows-x86_64-$VERSION"
   TARBALL="$DIRNAME.zip"
-  mv release "$DIRNAME"
+  mv dist "$DIRNAME"
   7z a "$TARBALL" "$DIRNAME"
 
   mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"

lib/libc/include/any-windows-any/psdk_inc/intrin-impl.h

@@ -1938,32 +1938,6 @@ __buildmov(__movsd, unsigned __LONG32, "d")
 #define __INTRINSIC_DEFINED___movsd
 #endif /* __INTRINSIC_PROLOG */
 
-#if !defined(__GNUC__) || __GNUC__ < 8 /* GCC 8 has already defined _xgetbv */
-/* NOTE: This should be in immintrin.h */
-#if __INTRINSIC_PROLOG(_xgetbv)
-unsigned __int64 _xgetbv(unsigned int);
-#if !__has_builtin(_xgetbv)
-__INTRINSICS_USEINLINE
-unsigned __int64 _xgetbv(unsigned int index)
-{
-#if defined(__x86_64__) || defined(_AMD64_)
-   unsigned __int64 val1, val2;
-#else
-   unsigned __LONG32 val1, val2;
-#endif /* defined(__x86_64__) || defined(_AMD64_) */
-
-   __asm__ __volatile__(
-      "xgetbv"
-      : "=a" (val1), "=d" (val2)
-      : "c" (index));
-
-   return (((unsigned __int64)val2) << 32) | val1;
-}
-#endif
-#define __INTRINSIC_DEFINED__xgetbv
-#endif /* __INTRINSIC_PROLOG */
-#endif /* __GNUC__ < 8 */
-
 #endif /* defined(__x86_64__) || defined(_AMD64_) || defined(__i386__) || defined(_X86_) */
 
 /* ***************************************************** */

src/buffer.cpp

@@ -61,7 +61,6 @@ void buf_appendf(Buf *buf, const char *format, ...) {
 
 // these functions are not static inline so they can be better used as template parameters
 bool buf_eql_buf(Buf *buf, Buf *other) {
-    assert(buf->list.length);
     return buf_eql_mem(buf, buf_ptr(other), buf_len(other));
 }
 

src/glibc.cpp

@@ -50,7 +50,7 @@ Error glibc_load_metadata(ZigGLibCAbi **out_result, Buf *zig_lib_dir, bool verbo
     }
 
     {
-        SplitIterator it = memSplit(buf_to_slice(vers_txt_contents), str("\n"));
+        SplitIterator it = memSplit(buf_to_slice(vers_txt_contents), str("\r\n"));
         for (;;) {
             Optional<Slice<uint8_t>> opt_component = SplitIterator_next(&it);
             if (!opt_component.is_some) break;
@@ -65,7 +65,7 @@ Error glibc_load_metadata(ZigGLibCAbi **out_result, Buf *zig_lib_dir, bool verbo
         }
     }
     {
-        SplitIterator it = memSplit(buf_to_slice(fns_txt_contents), str("\n"));
+        SplitIterator it = memSplit(buf_to_slice(fns_txt_contents), str("\r\n"));
         for (;;) {
             Optional<Slice<uint8_t>> opt_component = SplitIterator_next(&it);
             if (!opt_component.is_some) break;
@@ -91,17 +91,19 @@ Error glibc_load_metadata(ZigGLibCAbi **out_result, Buf *zig_lib_dir, bool verbo
         }
     }
     {
-        SplitIterator it = memSplit(buf_to_slice(abi_txt_contents), str("\n"));
+        SplitIterator it = memSplit(buf_to_slice(abi_txt_contents), str("\r\n"));
         ZigGLibCVerList *ver_list_base = nullptr;
+        int line_num = 0;
         for (;;) {
             if (ver_list_base == nullptr) {
+                line_num += 1;
                 Optional<Slice<uint8_t>> opt_line = SplitIterator_next_separate(&it);
                 if (!opt_line.is_some) break;
 
                 ver_list_base = allocate<ZigGLibCVerList>(glibc_abi->all_functions.length);
-                ZigTarget *target = allocate<ZigTarget>(1);
                 SplitIterator line_it = memSplit(opt_line.value, str(" "));
                 for (;;) {
+                    ZigTarget *target = allocate<ZigTarget>(1);
                     Optional<Slice<uint8_t>> opt_target = SplitIterator_next(&line_it);
                     if (!opt_target.is_some) break;
 
@@ -122,14 +124,27 @@ Error glibc_load_metadata(ZigGLibCAbi **out_result, Buf *zig_lib_dir, bool verbo
                     target->os = OsLinux;
 
                     err = target_parse_abi(&target->abi, (char*)opt_abi.value.ptr, opt_abi.value.len);
-                    assert(err == ErrorNone);
+                    if (err != ErrorNone) {
+                        fprintf(stderr, "Error parsing %s:%d: %s\n", buf_ptr(glibc_abi->abi_txt_path),
+                                line_num, err_str(err));
+                        fprintf(stderr, "arch: '%.*s', os: '%.*s', abi: '%.*s'\n",
+                                (int)opt_arch.value.len, (const char*)opt_arch.value.ptr,
+                                (int)opt_os.value.len, (const char*)opt_os.value.ptr,
+                                (int)opt_abi.value.len, (const char*)opt_abi.value.ptr);
+                        fprintf(stderr, "parsed from target: '%.*s'\n",
+                                (int)opt_target.value.len, (const char*)opt_target.value.ptr);
+                        fprintf(stderr, "parsed from line:\n%.*s\n", (int)opt_line.value.len, opt_line.value.ptr);
+                        fprintf(stderr, "Zig installation appears to be corrupted.\n");
+                        exit(1);
+                    }
 
                     glibc_abi->version_table.put(target, ver_list_base);
                 }
                 continue;
             }
             for (size_t fn_i = 0; fn_i < glibc_abi->all_functions.length; fn_i += 1) {
                 ZigGLibCVerList *ver_list = &ver_list_base[fn_i];
+                line_num += 1;
                 Optional<Slice<uint8_t>> opt_line = SplitIterator_next_separate(&it);
                 assert(opt_line.is_some);
 
@@ -266,7 +281,7 @@ Error glibc_build_dummies_and_maps(CodeGen *g, const ZigGLibCAbi *glibc_abi, con
             // - If there are no versions, don't emit it
             // - Take the greatest one <= than the target one
             // - If none of them is <= than the
-            //   specified one don't pick any default version 
+            //   specified one don't pick any default version
             if (ver_list->len == 0) continue;
             uint8_t chosen_def_ver_index = 255;
             for (uint8_t ver_i = 0; ver_i < ver_list->len; ver_i += 1) {
@@ -322,7 +337,7 @@ Error glibc_build_dummies_and_maps(CodeGen *g, const ZigGLibCAbi *glibc_abi, con
         codegen_set_lib_version(child_gen, lib->sover, 0, 0);
         child_gen->is_dynamic = true;
         child_gen->is_dummy_so = true;
-        child_gen->version_script_path = map_file_path; 
+        child_gen->version_script_path = map_file_path;
         child_gen->enable_cache = false;
         child_gen->output_dir = dummy_dir;
         codegen_build_and_link(child_gen);

src/link.cpp

@@ -1295,6 +1295,7 @@ static const char *get_libc_crt_file(CodeGen *parent, const char *file) {
             c_file->args.append("-include");
             c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h"));
             c_file->args.append("-DMODULE_NAME=libc");
+            c_file->args.append("-Wno-nonportable-include-path");
             c_file->args.append("-include");
             c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h"));
             c_file->args.append("-DTOP_NAMESPACE=glibc");
@@ -1321,6 +1322,7 @@ static const char *get_libc_crt_file(CodeGen *parent, const char *file) {
             c_file->args.append("-include");
             c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h"));
             c_file->args.append("-DMODULE_NAME=libc");
+            c_file->args.append("-Wno-nonportable-include-path");
             c_file->args.append("-include");
             c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h"));
             c_file->args.append("-DPIC");
@@ -1377,6 +1379,7 @@ static const char *get_libc_crt_file(CodeGen *parent, const char *file) {
                 c_file->args.append("-include");
                 c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h"));
                 c_file->args.append("-DMODULE_NAME=libc");
+                c_file->args.append("-Wno-nonportable-include-path");
                 c_file->args.append("-include");
                 c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h"));
                 c_file->args.append("-DPIC");
@@ -1420,6 +1423,7 @@ static const char *get_libc_crt_file(CodeGen *parent, const char *file) {
                 c_file->args.append("-include");
                 c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h"));
                 c_file->args.append("-DMODULE_NAME=libc");
+                c_file->args.append("-Wno-nonportable-include-path");
                 c_file->args.append("-include");
                 c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h"));
                 c_file->args.append("-DPIC");

src/os.cpp

@@ -116,7 +116,7 @@ static void os_spawn_process_posix(ZigList<const char *> &args, Termination *ter
     pid_t pid;
     int rc = posix_spawnp(&pid, args.at(0), nullptr, nullptr, const_cast<char *const*>(argv), environ);
     if (rc != 0) {
-        zig_panic("posix_spawn failed: %s", strerror(rc));
+        zig_panic("unable to spawn %s: %s", args.at(0), strerror(rc));
     }
 
     int status;

src/util.cpp

@@ -18,17 +18,10 @@ void zig_panic(const char *format, ...) {
     vfprintf(stderr, format, ap);
     fflush(stderr);
     va_end(ap);
-    stage2_panic(nullptr, 0);
+    stage2_panic("", 0);
     abort();
 }
 
-void assert(bool ok) {
-    if (!ok) {
-        const char *msg = "Assertion failed. This is a bug in the Zig compiler.";
-        stage2_panic(msg, strlen(msg));
-    }
-}
-
 uint32_t int_hash(int i) {
     return (uint32_t)(i % UINT32_MAX);
 }

src/util.hpp

@@ -43,15 +43,21 @@ ATTRIBUTE_NORETURN
 ATTRIBUTE_PRINTF(1, 2)
 void zig_panic(const char *format, ...);
 
+static inline void zig_assert(bool ok, const char *file, int line, const char *func) {
+    if (!ok) {
+        zig_panic("Assertion failed at %s:%d in %s. This is a bug in the Zig compiler.", file, line, func);
+    }
+}
+
 #ifdef _WIN32
 #define __func__ __FUNCTION__
 #endif
 
-#define zig_unreachable() zig_panic("unreachable: %s:%s:%d", __FILE__, __func__, __LINE__)
+#define zig_unreachable() zig_panic("Unreachable at %s:%d in %s. This is a bug in the Zig compiler.", __FILE__, __LINE__, __func__)
 
 // Assertions in stage1 are always on, and they call zig @panic.
 #undef assert
-void assert(bool ok);
+#define assert(ok) zig_assert(ok, __FILE__, __LINE__, __func__)
 
 #if defined(_MSC_VER)
 static inline int clzll(unsigned long long mask) {