From e3bed8d81dfd7198dd4c496f19a6791e27e41f26 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Wed, 29 Dec 2021 20:49:16 -0700
Subject: [PATCH 01/17] stage2: introduce CacheMode

The two CacheMode values are `whole` and `incremental`.
`incremental` is what we had before; `whole` is new.
Whole cache mode uses everything as inputs to the cache hash;
and when a hit occurs it skips everything including linking.
This is ideal for when source files change rarely and for backends that
do not have good incremental compilation support, for example
compiler-rt or libc compiled with LLVM with optimizations on.
This is the main motivation for the additional mode, so that we can have
LLVM-optimized compiler-rt/libc builds, without waiting for the LLVM
backend every single time Zig is invoked.

Incremental cache mode hashes only the input file path and a few target
options, intentionally relying on collisions to locate already-existing
build artifacts which can then be incrementally updated.

The bespoke logic for caching stage1 backend build artifacts
is removed since we now have a global caching mechanism for
when we want to cache the entire compilation, *including* linking.
Previously we had to get "creative" with libs.txt and a special
byte in the hash id to communicate flags, so that when the cached
artifacts were re-linked, we had this information from stage1
even though we didn't actually run it. Now that `CacheMode.whole`
includes linking, this extra information does not need to be
preserved for cache hits. So although this changeset introduces
complexity, it also removes complexity.

The main trickiness here comes from the inherent differences between the
two modes: `incremental` wants a directory immediately to operate on,
while `whole` doesn't know the output directory until the compilation is
complete. This commit deals with this problem mostly inside `update()`,
where, on a cache miss, it replaces `zig_cache_artifact_directory` with a
temporary directory, and then renames it into place once the compilation is
complete.

Items remaining before this branch can be merged:

* [ ] make sure these things make it into the cache manifest:
  - @import files
  - @embedFile files
  - we already add dep files from c but make sure the main .c files make
    it in there too, not just the included files

* [ ] double check that the emit paths of other things besides the binary
  are working correctly.

* [ ] test `-fno-emit-bin` + `-fstage1`
* [ ] test `-femit-bin=foo` + `-fstage1`

* [ ] implib emit directory copies bin_file_emit directory in create() and needs
  to be adjusted to be overridden as well.

* [ ] make sure emit-h is handled correctly in the cache hash
* [ ] Cache: detect duplicate files added to the manifest

Some preliminary performance measurements of wall clock time and
peak RSS used:

stage1 behavior (1077 tests), llvm backend, release build:
 * cold global cache: 4.6s, 1.1 GiB
 * warm global cache: 3.4s, 980 MiB

stage2 master branch behavior (575 tests), llvm backend, release build:
 * cold global cache: 0.62s, 191 MiB
 * warm global cache: 0.40s, 128 MiB

stage2 this branch behavior (575 tests), llvm backend, release build:
 * cold global cache: 0.62s, 179 MiB
 * warm global cache: 0.27s, 90 MiB
---
 src/Compilation.zig | 589 ++++++++++++++++++++++++++++----------------
 src/Module.zig      |   2 +-
 src/glibc.zig       |   1 +
 src/libcxx.zig      |   2 +
 src/libtsan.zig     |   1 +
 src/libunwind.zig   |   1 +
 src/link.zig        |   8 +-
 src/link/Coff.zig   |   2 +
 src/link/Elf.zig    |   2 +
 src/link/MachO.zig  |   2 +
 src/link/Wasm.zig   |   2 +
 src/main.zig        |   4 -
 src/musl.zig        |   1 +
 src/stage1.zig      |   2 +-
 14 files changed, 401 insertions(+), 218 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index cd4794554e4e..8afb373699ff 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -41,8 +41,8 @@ gpa: Allocator,
 arena_state: std.heap.ArenaAllocator.State,
 bin_file: *link.File,
 c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
-stage1_lock: ?Cache.Lock = null,
-stage1_cache_manifest: *Cache.Manifest = undefined,
+/// This is a pointer to a local variable inside `update()`.
+whole_cache_manifest: ?*Cache.Manifest = null,
 
 link_error_flags: link.File.ErrorFlags = .{},
 
@@ -98,6 +98,9 @@ clang_argv: []const []const u8,
 cache_parent: *Cache,
 /// Path to own executable for invoking `zig clang`.
 self_exe_path: ?[]const u8,
+/// null means -fno-emit-bin. Contains the basename of the
+/// outputted binary file in case we don't know the directory yet.
+whole_bin_basename: ?[]const u8,
 zig_lib_directory: Directory,
 local_cache_directory: Directory,
 global_cache_directory: Directory,
@@ -612,6 +615,15 @@ pub const Directory = struct {
             return std.fs.path.joinZ(allocator, paths);
         }
     }
+
+    /// Whether or not the handle should be closed, or the path should be freed
+    /// is determined by usage, however this function is provided for convenience
+    /// if it happens to be what the caller needs.
+    pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
+        self.handle.close();
+        if (self.path) |p| gpa.free(p);
+        self.* = undefined;
+    }
 };
 
 pub const EmitLoc = struct {
@@ -631,6 +643,7 @@ pub const ClangPreprocessorMode = enum {
 };
 
 pub const SystemLib = link.SystemLib;
+pub const CacheMode = link.CacheMode;
 
 pub const InitOptions = struct {
     zig_lib_directory: Directory,
@@ -668,6 +681,7 @@ pub const InitOptions = struct {
     /// is externally modified - essentially anything other than zig-cache - then
     /// this flag would be set to disable this machinery to avoid false positives.
     disable_lld_caching: bool = false,
+    cache_mode: CacheMode = .incremental,
     object_format: ?std.Target.ObjectFormat = null,
     optimize_mode: std.builtin.Mode = .Debug,
     keep_source_files_loaded: bool = false,
@@ -885,6 +899,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             break :blk build_options.is_stage1;
         };
 
+        const cache_mode = if (use_stage1) CacheMode.whole else options.cache_mode;
+
         // Make a decision on whether to use LLVM or our own backend.
         const use_llvm = build_options.have_llvm and blk: {
             if (options.use_llvm) |explicit|
@@ -1219,18 +1235,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             // modified between incremental updates.
             var hash = cache.hash;
 
-            // Here we put the root source file path name, but *not* with addFile. We want the
-            // hash to be the same regardless of the contents of the source file, because
-            // incremental compilation will handle it, but we do want to namespace different
-            // source file names because they are likely different compilations and therefore this
-            // would be likely to cause cache hits.
-            hash.addBytes(main_pkg.root_src_path);
-            hash.addOptionalBytes(main_pkg.root_src_directory.path);
-            {
-                var local_arena = std.heap.ArenaAllocator.init(gpa);
-                defer local_arena.deinit();
-                var seen_table = std.AutoHashMap(*Package, void).init(local_arena.allocator());
-                try addPackageTableToCacheHash(&hash, &local_arena, main_pkg.table, &seen_table, .path_bytes);
+            switch (cache_mode) {
+                .incremental => {
+                    // Here we put the root source file path name, but *not* with addFile.
+                    // We want the hash to be the same regardless of the contents of the
+                    // source file, because incremental compilation will handle it, but we
+                    // do want to namespace different source file names because they are
+                    // likely different compilations and therefore this would be likely to
+                    // cause cache hits.
+                    hash.addBytes(main_pkg.root_src_path);
+                    hash.addOptionalBytes(main_pkg.root_src_directory.path);
+                    {
+                        var seen_table = std.AutoHashMap(*Package, void).init(arena);
+                        try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes);
+                    }
+                },
+                .whole => {
+                    // In this case, we postpone adding the input source file until
+                    // we create the cache manifest, in update(), because we want to
+                    // track it and packages as files.
+                },
             }
             hash.add(valgrind);
             hash.add(single_threaded);
@@ -1238,9 +1262,35 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             hash.add(use_llvm);
             hash.add(dll_export_fns);
             hash.add(options.is_test);
+            hash.add(options.test_evented_io);
+            hash.addOptionalBytes(options.test_filter);
+            hash.addOptionalBytes(options.test_name_prefix);
             hash.add(options.skip_linker_dependencies);
             hash.add(options.parent_compilation_link_libc);
 
+            // In the case of incremental cache mode, this `zig_cache_artifact_directory`
+            // is computed based on a hash of non-linker inputs, and it is where all
+            // build artifacts are stored (even while in-progress).
+            //
+            // For whole cache mode, it is still used for builtin.zig so that the file
+            // path to builtin.zig can remain consistent during a debugging session at
+            // runtime. However, we don't know where to put outputs from the linker
+            // or stage1 backend object files until the final cache hash, which is available
+            // after the compilation is complete.
+            //
+            // Therefore, in whole cache mode, we additionally create a temporary cache
+            // directory for these two kinds of build artifacts, and then rename it
+            // into place after the final hash is known. However, we don't want
+            // to create the temporary directory here, because in the case of a cache hit,
+            // this would have been wasted syscalls to make the directory and then not
+            // use it (or delete it).
+            //
+            // In summary, for whole cache mode, we simulate `-fno-emit-bin` in this
+            // function, and `zig_cache_artifact_directory` is *wrong* except for builtin.zig,
+            // and then at the beginning of `update()` when we find out whether we need
+            // a temporary directory, we patch up all the places that the incorrect
+            // `zig_cache_artifact_directory` was passed to various components of the compiler.
+
             const digest = hash.final();
             const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
             var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{});
@@ -1374,6 +1424,11 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 };
             }
 
+            switch (cache_mode) {
+                .whole => break :blk null,
+                .incremental => {},
+            }
+
             if (module) |zm| {
                 break :blk link.Emit{
                     .directory = zm.zig_cache_artifact_directory,
@@ -1425,6 +1480,18 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             };
         };
 
+        // This is so that when doing `CacheMode.whole`, the mechanism in update()
+        // can use it for communicating the result directory via `bin_file.emit`.
+        // This is used to distinguish between -fno-emit-bin and -femit-bin
+        // for `CacheMode.whole`.
+        const whole_bin_basename: ?[]const u8 = if (options.emit_bin) |x|
+            if (x.directory == null)
+                x.basename
+            else
+                null
+        else
+            null;
+
         var system_libs: std.StringArrayHashMapUnmanaged(SystemLib) = .{};
         errdefer system_libs.deinit(gpa);
         try system_libs.ensureTotalCapacity(gpa, options.system_lib_names.len);
@@ -1512,7 +1579,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             .skip_linker_dependencies = options.skip_linker_dependencies,
             .parent_compilation_link_libc = options.parent_compilation_link_libc,
             .each_lib_rpath = options.each_lib_rpath orelse options.is_native_os,
-            .disable_lld_caching = options.disable_lld_caching,
+            .cache_mode = cache_mode,
+            .disable_lld_caching = options.disable_lld_caching or cache_mode == .whole,
             .subsystem = options.subsystem,
             .is_test = options.is_test,
             .wasi_exec_model = wasi_exec_model,
@@ -1529,6 +1597,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             .local_cache_directory = options.local_cache_directory,
             .global_cache_directory = options.global_cache_directory,
             .bin_file = bin_file,
+            .whole_bin_basename = whole_bin_basename,
             .emit_asm = options.emit_asm,
             .emit_llvm_ir = options.emit_llvm_ir,
             .emit_llvm_bc = options.emit_llvm_bc,
@@ -1725,20 +1794,11 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
     return comp;
 }
 
-fn releaseStage1Lock(comp: *Compilation) void {
-    if (comp.stage1_lock) |*lock| {
-        lock.release();
-        comp.stage1_lock = null;
-    }
-}
-
 pub fn destroy(self: *Compilation) void {
     const optional_module = self.bin_file.options.module;
     self.bin_file.destroy();
     if (optional_module) |module| module.deinit();
 
-    self.releaseStage1Lock();
-
     const gpa = self.gpa;
     self.work_queue.deinit();
     self.anon_work_queue.deinit();
@@ -1815,22 +1875,135 @@ pub fn getTarget(self: Compilation) Target {
     return self.bin_file.options.target;
 }
 
+fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Directory) void {
+    if (directory.path) |p| comp.gpa.free(p);
+
+    // Restore the Module's previous zig_cache_artifact_directory
+    // This is only for cleanup purposes; Module.deinit calls close
+    // on the handle of zig_cache_artifact_directory.
+    if (comp.bin_file.options.module) |module| {
+        const builtin_pkg = module.main_pkg.table.get("builtin").?;
+        module.zig_cache_artifact_directory = builtin_pkg.root_src_directory;
+    }
+}
+
+fn cleanupTmpArtifactDirectory(
+    comp: *Compilation,
+    tmp_artifact_directory: *?Directory,
+    tmp_dir_sub_path: []const u8,
+) void {
+    comp.gpa.free(tmp_dir_sub_path);
+    if (tmp_artifact_directory.*) |*directory| {
+        directory.handle.close();
+        restorePrevZigCacheArtifactDirectory(comp, directory);
+    }
+}
+
 /// Detect changes to source files, perform semantic analysis, and update the output files.
-pub fn update(self: *Compilation) !void {
+pub fn update(comp: *Compilation) !void {
     const tracy_trace = trace(@src());
     defer tracy_trace.end();
 
-    self.clearMiscFailures();
+    comp.clearMiscFailures();
+
+    var man: Cache.Manifest = undefined;
+    defer if (comp.whole_cache_manifest != null) man.deinit();
+
+    var tmp_dir_sub_path: []const u8 = &.{};
+    var tmp_artifact_directory: ?Directory = null;
+    defer cleanupTmpArtifactDirectory(comp, &tmp_artifact_directory, tmp_dir_sub_path);
+
+    // If using the whole caching strategy, we check for *everything* up front, including
+    // C source files.
+    if (comp.bin_file.options.cache_mode == .whole) {
+        // We are about to obtain this lock, so here we give other processes a chance first.
+        comp.bin_file.releaseLock();
+
+        man = comp.cache_parent.obtain();
+        try comp.addNonIncrementalStuffToCacheManifest(&man);
+
+        const is_hit = man.hit() catch |err| {
+            // TODO properly bubble these up instead of emitting a warning
+            const i = man.failed_file_index orelse return err;
+            const file_path = man.files.items[i].path orelse return err;
+            std.log.warn("{s}: {s}", .{ @errorName(err), file_path });
+            return err;
+        };
+        if (is_hit) {
+            const digest = man.final();
+
+            // Communicate the output binary location to parent Compilations.
+            if (comp.whole_bin_basename) |basename| {
+                const new_sub_path = try std.fs.path.join(comp.gpa, &.{
+                    "o", &digest, basename,
+                });
+                if (comp.bin_file.options.emit) |emit| {
+                    comp.gpa.free(emit.sub_path);
+                }
+                comp.bin_file.options.emit = .{
+                    .directory = comp.local_cache_directory,
+                    .sub_path = new_sub_path,
+                };
+            }
+
+            comp.emitOthers();
+
+            assert(comp.bin_file.lock == null);
+            comp.bin_file.lock = man.toOwnedLock();
+            return;
+        }
+        comp.whole_cache_manifest = &man;
+
+        // Initialize `bin_file.emit` with a temporary Directory so that compilation can
+        // continue on the same path as incremental, using the temporary Directory.
+        tmp_artifact_directory = d: {
+            const s = std.fs.path.sep_str;
+            const rand_int = std.crypto.random.int(u64);
+
+            tmp_dir_sub_path = try std.fmt.allocPrint(comp.gpa, "tmp" ++ s ++ "{x}", .{rand_int});
+
+            const path = try comp.local_cache_directory.join(comp.gpa, &.{tmp_dir_sub_path});
+            errdefer comp.gpa.free(path);
+
+            const handle = try comp.local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
+            errdefer handle.close();
+
+            break :d .{
+                .path = path,
+                .handle = handle,
+            };
+        };
+
+        // This updates the output directory for stage1 backend and linker outputs.
+        if (comp.bin_file.options.module) |module| {
+            module.zig_cache_artifact_directory = tmp_artifact_directory.?;
+        }
+
+        // This resets the link.File to operate as if we called openPath() in create()
+        // instead of simulating -fno-emit-bin.
+        var options = comp.bin_file.options;
+        if (comp.whole_bin_basename) |basename| {
+            if (options.emit) |emit| {
+                comp.gpa.free(emit.sub_path);
+            }
+            options.emit = .{
+                .directory = tmp_artifact_directory.?,
+                .sub_path = basename,
+            };
+        }
+        comp.bin_file.destroy();
+        comp.bin_file = try link.File.openPath(comp.gpa, options);
+    }
 
     // For compiling C objects, we rely on the cache hash system to avoid duplicating work.
     // Add a Job for each C object.
-    try self.c_object_work_queue.ensureUnusedCapacity(self.c_object_table.count());
-    for (self.c_object_table.keys()) |key| {
-        self.c_object_work_queue.writeItemAssumeCapacity(key);
+    try comp.c_object_work_queue.ensureUnusedCapacity(comp.c_object_table.count());
+    for (comp.c_object_table.keys()) |key| {
+        comp.c_object_work_queue.writeItemAssumeCapacity(key);
     }
 
-    const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_stage1;
-    if (self.bin_file.options.module) |module| {
+    const use_stage1 = build_options.is_stage1 and comp.bin_file.options.use_stage1;
+    if (comp.bin_file.options.module) |module| {
         module.compile_log_text.shrinkAndFree(module.gpa, 0);
         module.generation += 1;
 
@@ -1845,7 +2018,7 @@ pub fn update(self: *Compilation) !void {
         // import_table here.
         // Likewise, in the case of `zig test`, the test runner is the root source file,
         // and so there is nothing to import the main file.
-        if (use_stage1 or self.bin_file.options.is_test) {
+        if (use_stage1 or comp.bin_file.options.is_test) {
             _ = try module.importPkg(module.main_pkg);
         }
 
@@ -1854,34 +2027,34 @@ pub fn update(self: *Compilation) !void {
         // to update it.
         // We still want AstGen work items for stage1 so that we expose compile errors
         // that are implemented in stage2 but not stage1.
-        try self.astgen_work_queue.ensureUnusedCapacity(module.import_table.count());
+        try comp.astgen_work_queue.ensureUnusedCapacity(module.import_table.count());
         for (module.import_table.values()) |value| {
-            self.astgen_work_queue.writeItemAssumeCapacity(value);
+            comp.astgen_work_queue.writeItemAssumeCapacity(value);
         }
 
         if (!use_stage1) {
             // Put a work item in for checking if any files used with `@embedFile` changed.
             {
-                try self.embed_file_work_queue.ensureUnusedCapacity(module.embed_table.count());
+                try comp.embed_file_work_queue.ensureUnusedCapacity(module.embed_table.count());
                 var it = module.embed_table.iterator();
                 while (it.next()) |entry| {
                     const embed_file = entry.value_ptr.*;
-                    self.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
+                    comp.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
                 }
             }
 
-            try self.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
-            if (self.bin_file.options.is_test) {
-                try self.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
+            try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
+            if (comp.bin_file.options.is_test) {
+                try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
             }
         }
     }
 
-    try self.performAllTheWork();
+    try comp.performAllTheWork();
 
     if (!use_stage1) {
-        if (self.bin_file.options.module) |module| {
-            if (self.bin_file.options.is_test and self.totalErrorCount() == 0) {
+        if (comp.bin_file.options.module) |module| {
+            if (comp.bin_file.options.is_test and comp.totalErrorCount() == 0) {
                 // The `test_functions` decl has been intentionally postponed until now,
                 // at which point we must populate it with the list of test functions that
                 // have been discovered and not filtered out.
@@ -1910,39 +2083,184 @@ pub fn update(self: *Compilation) !void {
         }
     }
 
-    if (self.totalErrorCount() != 0) {
+    if (comp.totalErrorCount() != 0) {
         // Skip flushing.
-        self.link_error_flags = .{};
+        comp.link_error_flags = .{};
         return;
     }
 
     // This is needed before reading the error flags.
-    try self.bin_file.flush(self);
-    self.link_error_flags = self.bin_file.errorFlags();
+    try comp.bin_file.flush(comp);
+    comp.link_error_flags = comp.bin_file.errorFlags();
 
     if (!use_stage1) {
-        if (self.bin_file.options.module) |module| {
+        if (comp.bin_file.options.module) |module| {
             try link.File.C.flushEmitH(module);
         }
     }
 
     // Flush takes care of -femit-bin, but we still have -femit-llvm-ir, -femit-llvm-bc, and
     // -femit-asm to handle, in the case of C objects.
-    self.emitOthers();
+    comp.emitOthers();
 
     // If there are any errors, we anticipate the source files being loaded
     // to report error messages. Otherwise we unload all source files to save memory.
     // The ZIR needs to stay loaded in memory because (1) Decl objects contain references
     // to it, and (2) generic instantiations, comptime calls, inline calls will need
     // to reference the ZIR.
-    if (self.totalErrorCount() == 0 and !self.keep_source_files_loaded) {
-        if (self.bin_file.options.module) |module| {
+    if (!comp.keep_source_files_loaded) {
+        if (comp.bin_file.options.module) |module| {
             for (module.import_table.values()) |file| {
-                file.unloadTree(self.gpa);
-                file.unloadSource(self.gpa);
+                file.unloadTree(comp.gpa);
+                file.unloadSource(comp.gpa);
+            }
+        }
+    }
+
+    if (comp.whole_cache_manifest != null) {
+        const digest = man.final();
+
+        // Rename the temporary directory into place.
+        var directory = tmp_artifact_directory.?;
+        tmp_artifact_directory = null;
+
+        directory.handle.close();
+        defer restorePrevZigCacheArtifactDirectory(comp, &directory);
+
+        const o_sub_path = try std.fs.path.join(comp.gpa, &[_][]const u8{ "o", &digest });
+        defer comp.gpa.free(o_sub_path);
+
+        try std.fs.rename(
+            comp.local_cache_directory.handle,
+            tmp_dir_sub_path,
+            comp.local_cache_directory.handle,
+            o_sub_path,
+        );
+
+        // Failure here only means an unnecessary cache miss.
+        man.writeManifest() catch |err| {
+            log.warn("failed to write cache manifest: {s}", .{@errorName(err)});
+        };
+
+        // Communicate the output binary location to parent Compilations.
+        if (comp.whole_bin_basename) |basename| {
+            comp.bin_file.options.emit = .{
+                .directory = comp.local_cache_directory,
+                .sub_path = try std.fs.path.join(comp.gpa, &.{ "o", &digest, basename }),
+            };
+        }
+
+        assert(comp.bin_file.lock == null);
+        comp.bin_file.lock = man.toOwnedLock();
+        return;
+    }
+}
+
+/// This is only observed at compile-time and used to emit a compile error
+/// to remind the programmer to update multiple related pieces of code that
+/// are in different locations. Bump this number when adding or deleting
+/// anything from the link cache manifest.
+pub const link_hash_implementation_version = 1;
+
+fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifest) !void {
+    const gpa = comp.gpa;
+    const target = comp.getTarget();
+
+    var arena_allocator = std.heap.ArenaAllocator.init(gpa);
+    errdefer arena_allocator.deinit();
+    const arena = arena_allocator.allocator();
+
+    comptime assert(link_hash_implementation_version == 1);
+
+    if (comp.bin_file.options.module) |mod| {
+        const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{
+            mod.main_pkg.root_src_path,
+        });
+        _ = try man.addFile(main_zig_file, null);
+        {
+            var seen_table = std.AutoHashMap(*Package, void).init(arena);
+
+            // Skip builtin.zig; it is useless as an input, and we don't want to have to
+            // write it before checking for a cache hit.
+            const builtin_pkg = mod.main_pkg.table.get("builtin").?;
+            try seen_table.put(builtin_pkg, {});
+
+            try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
+        }
+    }
+
+    try man.addOptionalFile(comp.bin_file.options.linker_script);
+    try man.addOptionalFile(comp.bin_file.options.version_script);
+    try man.addListOfFiles(comp.bin_file.options.objects);
+
+    for (comp.c_object_table.keys()) |key| {
+        _ = try man.addFile(key.src.src_path, null);
+        man.hash.addListOfBytes(key.src.extra_flags);
+    }
+
+    man.hash.addOptionalEmitLoc(comp.emit_asm);
+    man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
+    man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
+    man.hash.addOptionalEmitLoc(comp.emit_analysis);
+    man.hash.addOptionalEmitLoc(comp.emit_docs);
+
+    man.hash.addListOfBytes(comp.clang_argv);
+
+    man.hash.addOptional(comp.bin_file.options.stack_size_override);
+    man.hash.addOptional(comp.bin_file.options.image_base_override);
+    man.hash.addOptional(comp.bin_file.options.gc_sections);
+    man.hash.add(comp.bin_file.options.eh_frame_hdr);
+    man.hash.add(comp.bin_file.options.emit_relocs);
+    man.hash.add(comp.bin_file.options.rdynamic);
+    man.hash.addListOfBytes(comp.bin_file.options.lib_dirs);
+    man.hash.addListOfBytes(comp.bin_file.options.rpath_list);
+    man.hash.add(comp.bin_file.options.each_lib_rpath);
+    man.hash.add(comp.bin_file.options.skip_linker_dependencies);
+    man.hash.add(comp.bin_file.options.z_nodelete);
+    man.hash.add(comp.bin_file.options.z_notext);
+    man.hash.add(comp.bin_file.options.z_defs);
+    man.hash.add(comp.bin_file.options.z_origin);
+    man.hash.add(comp.bin_file.options.z_noexecstack);
+    man.hash.add(comp.bin_file.options.z_now);
+    man.hash.add(comp.bin_file.options.z_relro);
+    man.hash.add(comp.bin_file.options.include_compiler_rt);
+    if (comp.bin_file.options.link_libc) {
+        man.hash.add(comp.bin_file.options.libc_installation != null);
+        if (comp.bin_file.options.libc_installation) |libc_installation| {
+            man.hash.addBytes(libc_installation.crt_dir.?);
+            if (target.abi == .msvc) {
+                man.hash.addBytes(libc_installation.msvc_lib_dir.?);
+                man.hash.addBytes(libc_installation.kernel32_lib_dir.?);
             }
         }
+        man.hash.addOptionalBytes(comp.bin_file.options.dynamic_linker);
     }
+    man.hash.addOptionalBytes(comp.bin_file.options.soname);
+    man.hash.addOptional(comp.bin_file.options.version);
+    link.hashAddSystemLibs(&man.hash, comp.bin_file.options.system_libs);
+    man.hash.addOptional(comp.bin_file.options.allow_shlib_undefined);
+    man.hash.add(comp.bin_file.options.bind_global_refs_locally);
+    man.hash.add(comp.bin_file.options.tsan);
+    man.hash.addOptionalBytes(comp.bin_file.options.sysroot);
+    man.hash.add(comp.bin_file.options.linker_optimization);
+
+    // WASM specific stuff
+    man.hash.add(comp.bin_file.options.import_memory);
+    man.hash.addOptional(comp.bin_file.options.initial_memory);
+    man.hash.addOptional(comp.bin_file.options.max_memory);
+    man.hash.addOptional(comp.bin_file.options.global_base);
+
+    // Mach-O specific stuff
+    man.hash.addListOfBytes(comp.bin_file.options.framework_dirs);
+    man.hash.addListOfBytes(comp.bin_file.options.frameworks);
+
+    // COFF specific stuff
+    man.hash.addOptional(comp.bin_file.options.subsystem);
+    man.hash.add(comp.bin_file.options.tsaware);
+    man.hash.add(comp.bin_file.options.nxcompat);
+    man.hash.add(comp.bin_file.options.dynamicbase);
+    man.hash.addOptional(comp.bin_file.options.major_subsystem_version);
+    man.hash.addOptional(comp.bin_file.options.minor_subsystem_version);
 }
 
 fn emitOthers(comp: *Compilation) void {
@@ -2988,7 +3306,9 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
 
         const dep_basename = std.fs.path.basename(out_dep_path);
         try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);
-        if (build_options.is_stage1 and comp.bin_file.options.use_stage1) try comp.stage1_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename);
+        if (comp.whole_cache_manifest) |whole_cache_manifest| {
+            try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename);
+        }
 
         const digest = man.final();
         const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
@@ -3351,13 +3671,13 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
     };
 }
 
-pub fn tmpFilePath(comp: *Compilation, arena: Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 {
+pub fn tmpFilePath(comp: *Compilation, ally: Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 {
     const s = std.fs.path.sep_str;
     const rand_int = std.crypto.random.int(u64);
     if (comp.local_cache_directory.path) |p| {
-        return std.fmt.allocPrint(arena, "{s}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix });
+        return std.fmt.allocPrint(ally, "{s}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix });
     } else {
-        return std.fmt.allocPrint(arena, "tmp" ++ s ++ "{x}-{s}", .{ rand_int, suffix });
+        return std.fmt.allocPrint(ally, "tmp" ++ s ++ "{x}-{s}", .{ rand_int, suffix });
     }
 }
 
@@ -4424,6 +4744,7 @@ fn buildOutputFromZig(
         .global_cache_directory = comp.global_cache_directory,
         .local_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
         .main_pkg = &main_pkg,
@@ -4501,10 +4822,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         mod.main_pkg.root_src_path,
     });
     const zig_lib_dir = comp.zig_lib_directory.path.?;
-    const builtin_zig_path = try directory.join(arena, &[_][]const u8{"builtin.zig"});
     const target = comp.getTarget();
-    const id_symlink_basename = "stage1.id";
-    const libs_txt_basename = "libs.txt";
 
     // The include_compiler_rt stored in the bin file options here means that we need
     // compiler-rt symbols *somehow*. However, in the context of using the stage1 backend
@@ -4516,115 +4834,6 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     const include_compiler_rt = comp.bin_file.options.output_mode == .Obj and
         comp.bin_file.options.include_compiler_rt;
 
-    // We are about to obtain this lock, so here we give other processes a chance first.
-    comp.releaseStage1Lock();
-
-    // Unlike with the self-hosted Zig module, stage1 does not support incremental compilation,
-    // so we input all the zig source files into the cache hash system. We're going to keep
-    // the artifact directory the same, however, so we take the same strategy as linking
-    // does where we have a file which specifies the hash of the output directory so that we can
-    // skip the expensive compilation step if the hash matches.
-    var man = comp.cache_parent.obtain();
-    defer man.deinit();
-
-    _ = try man.addFile(main_zig_file, null);
-    {
-        var seen_table = std.AutoHashMap(*Package, void).init(arena_allocator.allocator());
-        try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = &man });
-    }
-    man.hash.add(comp.bin_file.options.valgrind);
-    man.hash.add(comp.bin_file.options.single_threaded);
-    man.hash.add(target.os.getVersionRange());
-    man.hash.add(comp.bin_file.options.dll_export_fns);
-    man.hash.add(comp.bin_file.options.function_sections);
-    man.hash.add(include_compiler_rt);
-    man.hash.add(comp.bin_file.options.is_test);
-    man.hash.add(comp.bin_file.options.emit != null);
-    man.hash.add(mod.emit_h != null);
-    if (mod.emit_h) |emit_h| {
-        man.hash.addEmitLoc(emit_h.loc);
-    }
-    man.hash.addOptionalEmitLoc(comp.emit_asm);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
-    man.hash.addOptionalEmitLoc(comp.emit_analysis);
-    man.hash.addOptionalEmitLoc(comp.emit_docs);
-    man.hash.add(comp.test_evented_io);
-    man.hash.addOptionalBytes(comp.test_filter);
-    man.hash.addOptionalBytes(comp.test_name_prefix);
-    man.hash.addListOfBytes(comp.clang_argv);
-
-    // Capture the state in case we come back from this branch where the hash doesn't match.
-    const prev_hash_state = man.hash.peekBin();
-    const input_file_count = man.files.items.len;
-
-    const hit = man.hit() catch |err| {
-        const i = man.failed_file_index orelse return err;
-        const file_path = man.files.items[i].path orelse return err;
-        fatal("unable to build stage1 zig object: {s}: {s}", .{ @errorName(err), file_path });
-    };
-    if (hit) {
-        const digest = man.final();
-
-        // We use an extra hex-encoded byte here to store some flags.
-        var prev_digest_buf: [digest.len + 2]u8 = undefined;
-        const prev_digest: []u8 = Cache.readSmallFile(
-            directory.handle,
-            id_symlink_basename,
-            &prev_digest_buf,
-        ) catch |err| blk: {
-            log.debug("stage1 {s} new_digest={s} error: {s}", .{
-                mod.main_pkg.root_src_path,
-                std.fmt.fmtSliceHexLower(&digest),
-                @errorName(err),
-            });
-            // Handle this as a cache miss.
-            break :blk prev_digest_buf[0..0];
-        };
-        if (prev_digest.len >= digest.len + 2) hit: {
-            if (!mem.eql(u8, prev_digest[0..digest.len], &digest))
-                break :hit;
-
-            log.debug("stage1 {s} digest={s} match - skipping invocation", .{
-                mod.main_pkg.root_src_path,
-                std.fmt.fmtSliceHexLower(&digest),
-            });
-            var flags_bytes: [1]u8 = undefined;
-            _ = std.fmt.hexToBytes(&flags_bytes, prev_digest[digest.len..]) catch {
-                log.warn("bad cache stage1 digest: '{s}'", .{std.fmt.fmtSliceHexLower(prev_digest)});
-                break :hit;
-            };
-
-            if (directory.handle.readFileAlloc(comp.gpa, libs_txt_basename, 10 * 1024 * 1024)) |libs_txt| {
-                var it = mem.tokenize(u8, libs_txt, "\n");
-                while (it.next()) |lib_name| {
-                    try comp.stage1AddLinkLib(lib_name);
-                }
-            } else |err| switch (err) {
-                error.FileNotFound => {}, // That's OK, it just means 0 libs.
-                else => {
-                    log.warn("unable to read cached list of link libs: {s}", .{@errorName(err)});
-                    break :hit;
-                },
-            }
-            comp.stage1_lock = man.toOwnedLock();
-            mod.stage1_flags = @bitCast(@TypeOf(mod.stage1_flags), flags_bytes[0]);
-            return;
-        }
-        log.debug("stage1 {s} prev_digest={s} new_digest={s}", .{
-            mod.main_pkg.root_src_path,
-            std.fmt.fmtSliceHexLower(prev_digest),
-            std.fmt.fmtSliceHexLower(&digest),
-        });
-        man.unhit(prev_hash_state, input_file_count);
-    }
-
-    // We are about to change the output file to be different, so we invalidate the build hash now.
-    directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
-        error.FileNotFound => {},
-        else => |e| return e,
-    };
-
     const stage2_target = try arena.create(stage1.Stage2Target);
     stage2_target.* = .{
         .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch
@@ -4637,9 +4846,9 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         .llvm_target_abi = if (target_util.llvmMachineAbi(target)) |s| s.ptr else null,
     };
 
-    comp.stage1_cache_manifest = &man;
-
     const main_pkg_path = mod.main_pkg.root_src_directory.path orelse "";
+    const builtin_pkg = mod.main_pkg.table.get("builtin").?;
+    const builtin_zig_path = try builtin_pkg.root_src_directory.join(arena, &.{builtin_pkg.root_src_path});
 
     const stage1_module = stage1.create(
         @enumToInt(comp.bin_file.options.optimize_mode),
@@ -4740,19 +4949,8 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         .have_dllmain_crt_startup = false,
     };
 
-    const inferred_lib_start_index = comp.bin_file.options.system_libs.count();
     stage1_module.build_object();
 
-    if (comp.bin_file.options.system_libs.count() > inferred_lib_start_index) {
-        // We need to save the inferred link libs to the cache, otherwise if we get a cache hit
-        // next time we will be missing these libs.
-        var libs_txt = std.ArrayList(u8).init(arena);
-        for (comp.bin_file.options.system_libs.keys()[inferred_lib_start_index..]) |key| {
-            try libs_txt.writer().print("{s}\n", .{key});
-        }
-        try directory.handle.writeFile(libs_txt_basename, libs_txt.items);
-    }
-
     mod.stage1_flags = .{
         .have_c_main = stage1_module.have_c_main,
         .have_winmain = stage1_module.have_winmain,
@@ -4763,34 +4961,6 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     };
 
     stage1_module.destroy();
-
-    const digest = man.final();
-
-    // Update the small file with the digest. If it fails we can continue; it only
-    // means that the next invocation will have an unnecessary cache miss.
-    const stage1_flags_byte = @bitCast(u8, mod.stage1_flags);
-    log.debug("stage1 {s} final digest={s} flags={x}", .{
-        mod.main_pkg.root_src_path, std.fmt.fmtSliceHexLower(&digest), stage1_flags_byte,
-    });
-    var digest_plus_flags: [digest.len + 2]u8 = undefined;
-    digest_plus_flags[0..digest.len].* = digest;
-    assert(std.fmt.formatIntBuf(digest_plus_flags[digest.len..], stage1_flags_byte, 16, .lower, .{
-        .width = 2,
-        .fill = '0',
-    }) == 2);
-    log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{
-        digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup,
-    });
-    Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest_plus_flags) catch |err| {
-        log.warn("failed to save stage1 hash digest file: {s}", .{@errorName(err)});
-    };
-    // Failure here only means an unnecessary cache miss.
-    man.writeManifest() catch |err| {
-        log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
-    };
-    // We hang on to this lock so that the output file path can be used without
-    // other processes clobbering it.
-    comp.stage1_lock = man.toOwnedLock();
 }
 
 fn stage1LocPath(arena: Allocator, opt_loc: ?EmitLoc, cache_directory: Directory) ![]const u8 {
@@ -4862,6 +5032,7 @@ pub fn build_crt_file(
         .local_cache_directory = comp.global_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
         .main_pkg = null,
diff --git a/src/Module.zig b/src/Module.zig
index ef01aa2c549d..fa79783c6af8 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -33,7 +33,7 @@ const build_options = @import("build_options");
 gpa: Allocator,
 comp: *Compilation,
 
-/// Where our incremental compilation metadata serialization will go.
+/// Where build artifacts and incremental compilation metadata serialization go.
 zig_cache_artifact_directory: Compilation.Directory,
 /// Pointer to externally managed resource.
 root_pkg: *Package,
diff --git a/src/glibc.zig b/src/glibc.zig
index e9c0651fdcd9..4424cb9b9022 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -1062,6 +1062,7 @@ fn buildSharedLib(
         .local_cache_directory = zig_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
         .target = comp.getTarget(),
         .root_name = lib.name,
         .main_pkg = null,
diff --git a/src/libcxx.zig b/src/libcxx.zig
index fe96207c483a..42ac1bf6ad15 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -177,6 +177,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .local_cache_directory = comp.global_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
         .main_pkg = null,
@@ -309,6 +310,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .local_cache_directory = comp.global_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
         .main_pkg = null,
diff --git a/src/libtsan.zig b/src/libtsan.zig
index 0f05957387ff..c7f288e76fad 100644
--- a/src/libtsan.zig
+++ b/src/libtsan.zig
@@ -199,6 +199,7 @@ pub fn buildTsan(comp: *Compilation) !void {
         .local_cache_directory = comp.global_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
         .main_pkg = null,
diff --git a/src/libunwind.zig b/src/libunwind.zig
index 95c58936fa6d..101ca6c8d404 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -101,6 +101,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .local_cache_directory = comp.global_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
         .main_pkg = null,
diff --git a/src/link.zig b/src/link.zig
index 464d96ddaca8..215d24adca53 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -22,6 +22,8 @@ pub const SystemLib = struct {
     needed: bool = false,
 };
 
+pub const CacheMode = enum { incremental, whole };
+
 pub fn hashAddSystemLibs(
     hh: *Cache.HashHelper,
     hm: std.StringArrayHashMapUnmanaged(SystemLib),
@@ -44,10 +46,9 @@ pub const Emit = struct {
 };
 
 pub const Options = struct {
-    /// This is `null` when -fno-emit-bin is used. When `openPath` or `flush` is called,
-    /// it will have already been null-checked.
+    /// This is `null` when `-fno-emit-bin` is used.
     emit: ?Emit,
-    /// This is `null` not building a Windows DLL, or when -fno-emit-implib is used.
+    /// This is `null` not building a Windows DLL, or when `-fno-emit-implib` is used.
     implib_emit: ?Emit,
     target: std.Target,
     output_mode: std.builtin.OutputMode,
@@ -70,6 +71,7 @@ pub const Options = struct {
     entry_addr: ?u64 = null,
     stack_size_override: ?u64,
     image_base_override: ?u64,
+    cache_mode: CacheMode,
     include_compiler_rt: bool,
     /// Set to `true` to omit debug info.
     strip: bool,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 2445b11caf0b..e6788120a050 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -920,6 +920,8 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         man = comp.cache_parent.obtain();
         self.base.releaseLock();
 
+        comptime assert(Compilation.link_hash_implementation_version == 1);
+
         try man.addListOfFiles(self.base.options.objects);
         for (comp.c_object_table.keys()) |key| {
             _ = try man.addFile(key.status.success.object_path, null);
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 992332be36e9..5f3771a3cb5b 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1357,6 +1357,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         // We are about to obtain this lock, so here we give other processes a chance first.
         self.base.releaseLock();
 
+        comptime assert(Compilation.link_hash_implementation_version == 1);
+
         try man.addOptionalFile(self.base.options.linker_script);
         try man.addOptionalFile(self.base.options.version_script);
         try man.addListOfFiles(self.base.options.objects);
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 7dc6c52372dc..f970ffb5f206 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -466,6 +466,8 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
         // We are about to obtain this lock, so here we give other processes a chance first.
         self.base.releaseLock();
 
+        comptime assert(Compilation.link_hash_implementation_version == 1);
+
         try man.addListOfFiles(self.base.options.objects);
         for (comp.c_object_table.keys()) |key| {
             _ = try man.addFile(key.status.success.object_path, null);
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
index 1ae50e8e3353..5f8f48d1b19a 100644
--- a/src/link/Wasm.zig
+++ b/src/link/Wasm.zig
@@ -1094,6 +1094,8 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
         // We are about to obtain this lock, so here we give other processes a chance first.
         self.base.releaseLock();
 
+        comptime assert(Compilation.link_hash_implementation_version == 1);
+
         try man.addListOfFiles(self.base.options.objects);
         for (comp.c_object_table.keys()) |key| {
             _ = try man.addFile(key.status.success.object_path, null);
diff --git a/src/main.zig b/src/main.zig
index 3f2dbd0cc797..53e7df352021 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -2578,10 +2578,6 @@ fn buildOutputType(
     };
     try comp.makeBinFileExecutable();
 
-    if (build_options.is_stage1 and comp.stage1_lock != null and watch) {
-        warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{});
-    }
-
     if (test_exec_args.items.len == 0 and object_format == .c) default_exec_args: {
         // Default to using `zig run` to execute the produced .c code from `zig test`.
         const c_code_loc = emit_bin_loc orelse break :default_exec_args;
diff --git a/src/musl.zig b/src/musl.zig
index d15d691a9c62..c576dbec8429 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -203,6 +203,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
             const sub_compilation = try Compilation.create(comp.gpa, .{
                 .local_cache_directory = comp.global_cache_directory,
                 .global_cache_directory = comp.global_cache_directory,
+                .cache_mode = .whole,
                 .zig_lib_directory = comp.zig_lib_directory,
                 .target = target,
                 .root_name = "c",
diff --git a/src/stage1.zig b/src/stage1.zig
index 8263427fbe6d..a5be1b78fe83 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -458,7 +458,7 @@ export fn stage2_fetch_file(
     const comp = @intToPtr(*Compilation, stage1.userdata);
     const file_path = path_ptr[0..path_len];
     const max_file_size = std.math.maxInt(u32);
-    const contents = comp.stage1_cache_manifest.addFilePostFetch(file_path, max_file_size) catch return null;
+    const contents = comp.whole_cache_manifest.?.addFilePostFetch(file_path, max_file_size) catch return null;
     result_len.* = contents.len;
     // TODO https://github.com/ziglang/zig/issues/3328#issuecomment-716749475
     if (contents.len == 0) return @intToPtr(?[*]const u8, 0x1);

From 67e31807df73ad2da992293fcdf1f6ea7ca67d2a Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 15:32:50 -0700
Subject: [PATCH 02/17] stage2: CacheMode.whole fixes

 * Logic to check whether a bin file is not emitted is more complicated
   in between `Compilation.create` and `Compilation.update`. Fixed the
   logic that decides whether to build compiler-rt and other support
   artifacts.
 * Basically, one cannot inspect the value of `comp.bin_file.emit` until
   after update() is called - fixed another instance of this happening
   in the CLI.
 * In the CLI, `runOrTest` is updated to properly use the result value
   of `comp.bin_file.options.emit` rather than guessing whether the
   output binary is.
 * Don't assume that the emit output has no directory components in
   sub_path. In other words, don't assume that the emit directory is the
   final directory; there may be sub-directories.
---
 src/Compilation.zig | 11 ++++++++++-
 src/main.zig        | 30 ++++++++++++++++--------------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 8afb373699ff..ac48804848e6 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1662,7 +1662,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});
     }
 
-    if (comp.bin_file.options.emit != null and !comp.bin_file.options.skip_linker_dependencies) {
+    const have_bin_emit = comp.bin_file.options.emit != null or comp.whole_bin_basename != null;
+
+    if (have_bin_emit and !comp.bin_file.options.skip_linker_dependencies) {
         // If we need to build glibc for the target, add work items for it.
         // We go through the work queue so that building can be done in parallel.
         if (comp.wantBuildGLibCFromSource()) {
@@ -1767,8 +1769,10 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
 
         if (comp.bin_file.options.include_compiler_rt and capable_of_building_compiler_rt) {
             if (is_exe_or_dyn_lib) {
+                log.debug("queuing a job to build compiler_rt_lib", .{});
                 try comp.work_queue.writeItem(.{ .compiler_rt_lib = {} });
             } else if (options.output_mode != .Obj) {
+                log.debug("queuing a job to build compiler_rt_obj", .{});
                 // If build-obj with -fcompiler-rt is requested, that is handled specially
                 // elsewhere. In this case we are making a static library, so we ask
                 // for a compiler-rt object to put in it.
@@ -1930,6 +1934,7 @@ pub fn update(comp: *Compilation) !void {
             return err;
         };
         if (is_hit) {
+            log.debug("CacheMode.whole cache hit for {s}", .{comp.bin_file.options.root_name});
             const digest = man.final();
 
             // Communicate the output binary location to parent Compilations.
@@ -1952,6 +1957,8 @@ pub fn update(comp: *Compilation) !void {
             comp.bin_file.lock = man.toOwnedLock();
             return;
         }
+        log.debug("CacheMode.whole cache miss for {s}", .{comp.bin_file.options.root_name});
+
         comp.whole_cache_manifest = &man;
 
         // Initialize `bin_file.emit` with a temporary Directory so that compilation can
@@ -2187,6 +2194,8 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
 
             try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
         }
+
+        man.hash.add(mod.emit_h != null);
     }
 
     try man.addOptionalFile(comp.bin_file.options.linker_script);
diff --git a/src/main.zig b/src/main.zig
index 53e7df352021..6f55e46eb1c2 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -2564,9 +2564,7 @@ fn buildOutputType(
 
         switch (emit_bin) {
             .no => break :blk .none,
-            .yes_default_path => break :blk .{
-                .print = comp.bin_file.options.emit.?.directory.path orelse ".",
-            },
+            .yes_default_path => break :blk .print_emit_bin_dir_path,
             .yes => |full_path| break :blk .{ .update = full_path },
             .yes_a_out => break :blk .{ .update = a_out_basename },
         }
@@ -2598,7 +2596,6 @@ fn buildOutputType(
             comp,
             gpa,
             arena,
-            emit_bin_loc,
             test_exec_args.items,
             self_exe_path,
             arg_mode,
@@ -2671,7 +2668,6 @@ fn buildOutputType(
                         comp,
                         gpa,
                         arena,
-                        emit_bin_loc,
                         test_exec_args.items,
                         self_exe_path,
                         arg_mode,
@@ -2697,7 +2693,6 @@ fn buildOutputType(
                         comp,
                         gpa,
                         arena,
-                        emit_bin_loc,
                         test_exec_args.items,
                         self_exe_path,
                         arg_mode,
@@ -2762,7 +2757,6 @@ fn runOrTest(
     comp: *Compilation,
     gpa: Allocator,
     arena: Allocator,
-    emit_bin_loc: ?Compilation.EmitLoc,
     test_exec_args: []const ?[]const u8,
     self_exe_path: []const u8,
     arg_mode: ArgMode,
@@ -2773,10 +2767,11 @@ fn runOrTest(
     runtime_args_start: ?usize,
     link_libc: bool,
 ) !void {
-    const exe_loc = emit_bin_loc orelse return;
-    const exe_directory = exe_loc.directory orelse comp.bin_file.options.emit.?.directory;
+    const exe_emit = comp.bin_file.options.emit orelse return;
+    // A naive `directory.join` here will indeed get the correct path to the binary,
+    // however, in the case of cwd, we actually want `./foo` so that the path can be executed.
     const exe_path = try fs.path.join(arena, &[_][]const u8{
-        exe_directory.path orelse ".", exe_loc.basename,
+        exe_emit.directory.path orelse ".", exe_emit.sub_path,
     });
 
     var argv = std.ArrayList([]const u8).init(gpa);
@@ -2880,7 +2875,7 @@ fn runOrTest(
 
 const AfterUpdateHook = union(enum) {
     none,
-    print: []const u8,
+    print_emit_bin_dir_path,
     update: []const u8,
 };
 
@@ -2906,7 +2901,13 @@ fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void
         return error.SemanticAnalyzeFail;
     } else switch (hook) {
         .none => {},
-        .print => |bin_path| try io.getStdOut().writer().print("{s}\n", .{bin_path}),
+        .print_emit_bin_dir_path => {
+            const emit = comp.bin_file.options.emit.?;
+            const full_path = try emit.directory.join(gpa, &.{emit.sub_path});
+            defer gpa.free(full_path);
+            const dir_path = fs.path.dirname(full_path).?;
+            try io.getStdOut().writer().print("{s}\n", .{dir_path});
+        },
         .update => |full_path| {
             const bin_sub_path = comp.bin_file.options.emit.?.sub_path;
             const cwd = fs.cwd();
@@ -3469,9 +3470,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
         };
         try comp.makeBinFileExecutable();
 
-        child_argv.items[argv_index_exe] = try comp.bin_file.options.emit.?.directory.join(
+        const emit = comp.bin_file.options.emit.?;
+        child_argv.items[argv_index_exe] = try emit.directory.join(
             arena,
-            &[_][]const u8{exe_basename},
+            &[_][]const u8{emit.sub_path},
         );
 
         break :argv child_argv.items;

From e36718165cdc29b777392a3a343d92ccd1c6acf3 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 16:42:32 -0700
Subject: [PATCH 03/17] stage2: add `@import` and `@embedFile` to CacheHash

when using `CacheMode.whole`. Also, I verified that `addDepFilePost` is
in fact including the original C source file in addition to the files it
depends on.
---
 src/Cache.zig  | 57 +++++++++++++++++++++++++++++++++++++++++++++++---
 src/Module.zig | 23 ++++++++++++++++++++
 2 files changed, 77 insertions(+), 3 deletions(-)

diff --git a/src/Cache.zig b/src/Cache.zig
index 94ad947f6992..a5995f64ea42 100644
--- a/src/Cache.zig
+++ b/src/Cache.zig
@@ -47,10 +47,16 @@ pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
 pub const File = struct {
     path: ?[]const u8,
     max_file_size: ?usize,
-    stat: fs.File.Stat,
+    stat: Stat,
     bin_digest: BinDigest,
     contents: ?[]const u8,
 
+    pub const Stat = struct {
+        inode: fs.File.INode,
+        size: u64,
+        mtime: i128,
+    };
+
     pub fn deinit(self: *File, allocator: Allocator) void {
         if (self.path) |owned_slice| {
             allocator.free(owned_slice);
@@ -424,7 +430,11 @@ pub const Manifest = struct {
             if (!size_match or !mtime_match or !inode_match) {
                 self.manifest_dirty = true;
 
-                cache_hash_file.stat = actual_stat;
+                cache_hash_file.stat = .{
+                    .size = actual_stat.size,
+                    .mtime = actual_stat.mtime,
+                    .inode = actual_stat.inode,
+                };
 
                 if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
                     // The actual file has an unreliable timestamp, force it to be hashed
@@ -530,7 +540,12 @@ pub const Manifest = struct {
         const file = try fs.cwd().openFile(ch_file.path.?, .{});
         defer file.close();
 
-        ch_file.stat = try file.stat();
+        const actual_stat = try file.stat();
+        ch_file.stat = .{
+            .size = actual_stat.size,
+            .mtime = actual_stat.mtime,
+            .inode = actual_stat.inode,
+        };
 
         if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
             // The actual file has an unreliable timestamp, force it to be hashed
@@ -615,6 +630,42 @@ pub const Manifest = struct {
         try self.populateFileHash(new_ch_file);
     }
 
+    /// Like `addFilePost` but when the file contents have already been loaded from disk.
+    /// On success, cache takes ownership of `resolved_path`.
+    pub fn addFilePostContents(
+        self: *Manifest,
+        resolved_path: []const u8,
+        bytes: []const u8,
+        stat: File.Stat,
+    ) error{OutOfMemory}!void {
+        assert(self.manifest_file != null);
+
+        const ch_file = try self.files.addOne(self.cache.gpa);
+        errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
+
+        ch_file.* = .{
+            .path = resolved_path,
+            .max_file_size = null,
+            .stat = stat,
+            .bin_digest = undefined,
+            .contents = null,
+        };
+
+        if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
+            // The actual file has an unreliable timestamp, force it to be hashed
+            ch_file.stat.mtime = 0;
+            ch_file.stat.inode = 0;
+        }
+
+        {
+            var hasher = hasher_init;
+            hasher.update(bytes);
+            hasher.final(&ch_file.bin_digest);
+        }
+
+        self.hash.hasher.update(&ch_file.bin_digest);
+    }
+
     pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
         assert(self.manifest_file != null);
 
diff --git a/src/Module.zig b/src/Module.zig
index fa79783c6af8..6742ddd48648 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -3380,6 +3380,19 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void {
             error.OutOfMemory => return error.OutOfMemory,
             error.AnalysisFail => {},
         }
+
+        if (mod.comp.whole_cache_manifest) |man| {
+            assert(file.source_loaded);
+            const resolved_path = try file.pkg.root_src_directory.join(gpa, &.{
+                file.sub_file_path,
+            });
+            errdefer gpa.free(resolved_path);
+            try man.addFilePostContents(resolved_path, file.source, .{
+                .size = file.stat_size,
+                .inode = file.stat_inode,
+                .mtime = file.stat_mtime,
+            });
+        }
     } else {
         new_decl.analysis = .file_failure;
     }
@@ -3836,6 +3849,16 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
         resolved_root_path, resolved_path, sub_file_path, rel_file_path,
     });
 
+    if (mod.comp.whole_cache_manifest) |man| {
+        const copied_resolved_path = try gpa.dupe(u8, resolved_path);
+        errdefer gpa.free(copied_resolved_path);
+        try man.addFilePostContents(copied_resolved_path, bytes, .{
+            .size = stat.size,
+            .inode = stat.inode,
+            .mtime = stat.mtime,
+        });
+    }
+
     keep_resolved_path = true; // It's now owned by embed_table.
     gop.value_ptr.* = new_file;
     new_file.* = .{

From d6c5602d4665ba4e9e7c0b7f42bd00b2489d420c Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 17:05:17 -0700
Subject: [PATCH 04/17] stage2: fix CLI not populating output binary files

This fixes a regression in this branch that can be reproduced with the
following steps:

1. `zig build-exe hello.zig`
2. delete the "hello" binary
3. `zig build-exe hello.zig`
4. observe that the "hello" binary is missing

This happened because it was a cache hit, but nothing got copied to the
output directory.

This commit sets CacheMode to incremental - even for stage1 - when the
CLI requests `disable_lld_caching` (this option should be renamed),
resulting in the main Compilation to be repeated (uncached) for stage1,
populating the binary into the cwd as expected.

For stage2 the result is even better: the incremental compilation system
will look for build artifacts to incrementally compile, and start fresh
if not found.
---
 src/Compilation.zig | 7 ++++---
 src/stage1.zig      | 5 ++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index ac48804848e6..ddf331d4f3ad 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -899,7 +899,10 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             break :blk build_options.is_stage1;
         };
 
-        const cache_mode = if (use_stage1) CacheMode.whole else options.cache_mode;
+        const cache_mode = if (use_stage1 and !options.disable_lld_caching)
+            CacheMode.whole
+        else
+            options.cache_mode;
 
         // Make a decision on whether to use LLVM or our own backend.
         const use_llvm = build_options.have_llvm and blk: {
@@ -1951,8 +1954,6 @@ pub fn update(comp: *Compilation) !void {
                 };
             }
 
-            comp.emitOthers();
-
             assert(comp.bin_file.lock == null);
             comp.bin_file.lock = man.toOwnedLock();
             return;
diff --git a/src/stage1.zig b/src/stage1.zig
index a5be1b78fe83..384988dc075b 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -458,7 +458,10 @@ export fn stage2_fetch_file(
     const comp = @intToPtr(*Compilation, stage1.userdata);
     const file_path = path_ptr[0..path_len];
     const max_file_size = std.math.maxInt(u32);
-    const contents = comp.whole_cache_manifest.?.addFilePostFetch(file_path, max_file_size) catch return null;
+    const contents = if (comp.whole_cache_manifest) |man|
+        man.addFilePostFetch(file_path, max_file_size) catch return null
+    else
+        std.fs.cwd().readFileAlloc(comp.gpa, file_path, max_file_size) catch return null;
     result_len.* = contents.len;
     // TODO https://github.com/ziglang/zig/issues/3328#issuecomment-716749475
     if (contents.len == 0) return @intToPtr(?[*]const u8, 0x1);

From dbd0a2c35d590752dfa586a816f2870e4bcb3200 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 17:54:40 -0700
Subject: [PATCH 05/17] stage2: fix path to cache artifacts in libcxx,

libtsan, libunwind, and libcxxabi.
---
 src/libcxx.zig    | 14 ++++++--------
 src/libtsan.zig   |  7 +++----
 src/libunwind.zig |  8 ++++----
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/libcxx.zig b/src/libcxx.zig
index 42ac1bf6ad15..2da680a40d29 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -219,10 +219,9 @@ pub fn buildLibCXX(comp: *Compilation) !void {
 
     assert(comp.libcxx_static_lib == null);
     comp.libcxx_static_lib = Compilation.CRTFile{
-        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(
-            comp.gpa,
-            &[_][]const u8{basename},
-        ),
+        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{
+            sub_compilation.bin_file.options.emit.?.sub_path,
+        }),
         .lock = sub_compilation.bin_file.toOwnedLock(),
     };
 }
@@ -352,10 +351,9 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
 
     assert(comp.libcxxabi_static_lib == null);
     comp.libcxxabi_static_lib = Compilation.CRTFile{
-        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(
-            comp.gpa,
-            &[_][]const u8{basename},
-        ),
+        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{
+            sub_compilation.bin_file.options.emit.?.sub_path,
+        }),
         .lock = sub_compilation.bin_file.toOwnedLock(),
     };
 }
diff --git a/src/libtsan.zig b/src/libtsan.zig
index c7f288e76fad..cbbcd11a8a08 100644
--- a/src/libtsan.zig
+++ b/src/libtsan.zig
@@ -238,10 +238,9 @@ pub fn buildTsan(comp: *Compilation) !void {
 
     assert(comp.tsan_static_lib == null);
     comp.tsan_static_lib = Compilation.CRTFile{
-        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(
-            comp.gpa,
-            &[_][]const u8{basename},
-        ),
+        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{
+            sub_compilation.bin_file.options.emit.?.sub_path,
+        }),
         .lock = sub_compilation.bin_file.toOwnedLock(),
     };
 }
diff --git a/src/libunwind.zig b/src/libunwind.zig
index 101ca6c8d404..0d030be32945 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -142,11 +142,11 @@ pub fn buildStaticLib(comp: *Compilation) !void {
     try sub_compilation.updateSubCompilation();
 
     assert(comp.libunwind_static_lib == null);
+
     comp.libunwind_static_lib = Compilation.CRTFile{
-        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(
-            comp.gpa,
-            &[_][]const u8{basename},
-        ),
+        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{
+            sub_compilation.bin_file.options.emit.?.sub_path,
+        }),
         .lock = sub_compilation.bin_file.toOwnedLock(),
     };
 }

From 0ad2a99675d331c686847a7b2a84feddfcce6573 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 20:49:02 -0700
Subject: [PATCH 06/17] stage2: CacheMode.whole: trigger loading zig source
 files

Previously the code asserted source files were already loaded, but this
is not the case when cached ZIR is loaded. Now it will trigger .zig
source code to be loaded for the purposes of hashing the source for
`CacheMode.whole`.

This additionally refactors stat_size, stat_inode, and stat_mtime fields
into using the `Cache.File.Stat` struct.
---
 src/Compilation.zig |   4 +-
 src/Module.zig      | 156 ++++++++++++++++++++++++++++----------------
 src/main.zig        |  34 +++++-----
 3 files changed, 121 insertions(+), 73 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index ddf331d4f3ad..e71c84af3377 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -421,7 +421,7 @@ pub const AllErrors = struct {
             const module_note = module_err_msg.notes[i];
             const source = try module_note.src_loc.file_scope.getSource(module.gpa);
             const byte_offset = try module_note.src_loc.byteOffset(module.gpa);
-            const loc = std.zig.findLineColumn(source, byte_offset);
+            const loc = std.zig.findLineColumn(source.bytes, byte_offset);
             const file_path = try module_note.src_loc.file_scope.fullPath(allocator);
             note.* = .{
                 .src = .{
@@ -444,7 +444,7 @@ pub const AllErrors = struct {
         }
         const source = try module_err_msg.src_loc.file_scope.getSource(module.gpa);
         const byte_offset = try module_err_msg.src_loc.byteOffset(module.gpa);
-        const loc = std.zig.findLineColumn(source, byte_offset);
+        const loc = std.zig.findLineColumn(source.bytes, byte_offset);
         const file_path = try module_err_msg.src_loc.file_scope.fullPath(allocator);
         try errors.append(.{
             .src = .{
diff --git a/src/Module.zig b/src/Module.zig
index 6742ddd48648..0cbf75c735b0 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -1463,11 +1463,7 @@ pub const File = struct {
     /// Whether this is populated depends on `source_loaded`.
     source: [:0]const u8,
     /// Whether this is populated depends on `status`.
-    stat_size: u64,
-    /// Whether this is populated depends on `status`.
-    stat_inode: std.fs.File.INode,
-    /// Whether this is populated depends on `status`.
-    stat_mtime: i128,
+    stat: Cache.File.Stat,
     /// Whether this is populated or not depends on `tree_loaded`.
     tree: Ast,
     /// Whether this is populated or not depends on `zir_loaded`.
@@ -1535,8 +1531,16 @@ pub const File = struct {
         file.* = undefined;
     }
 
-    pub fn getSource(file: *File, gpa: Allocator) ![:0]const u8 {
-        if (file.source_loaded) return file.source;
+    pub const Source = struct {
+        bytes: [:0]const u8,
+        stat: Cache.File.Stat,
+    };
+
+    pub fn getSource(file: *File, gpa: Allocator) !Source {
+        if (file.source_loaded) return Source{
+            .bytes = file.source,
+            .stat = file.stat,
+        };
 
         const root_dir_path = file.pkg.root_src_directory.path orelse ".";
         log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{
@@ -1565,14 +1569,21 @@ pub const File = struct {
 
         file.source = source;
         file.source_loaded = true;
-        return source;
+        return Source{
+            .bytes = source,
+            .stat = .{
+                .size = stat.size,
+                .inode = stat.inode,
+                .mtime = stat.mtime,
+            },
+        };
     }
 
     pub fn getTree(file: *File, gpa: Allocator) !*const Ast {
         if (file.tree_loaded) return &file.tree;
 
         const source = try file.getSource(gpa);
-        file.tree = try std.zig.parse(gpa, source);
+        file.tree = try std.zig.parse(gpa, source.bytes);
         file.tree_loaded = true;
         return &file.tree;
     }
@@ -1631,9 +1642,7 @@ pub const EmbedFile = struct {
     /// Memory is stored in gpa, owned by EmbedFile.
     sub_file_path: []const u8,
     bytes: [:0]const u8,
-    stat_size: u64,
-    stat_inode: std.fs.File.INode,
-    stat_mtime: i128,
+    stat: Cache.File.Stat,
     /// Package that this file is a part of, managed externally.
     pkg: *Package,
     /// The Decl that was created from the `@embedFile` to own this resource.
@@ -2704,9 +2713,11 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
             keep_zir = true;
             file.zir = zir;
             file.zir_loaded = true;
-            file.stat_size = header.stat_size;
-            file.stat_inode = header.stat_inode;
-            file.stat_mtime = header.stat_mtime;
+            file.stat = .{
+                .size = header.stat_size,
+                .inode = header.stat_inode,
+                .mtime = header.stat_mtime,
+            };
             file.status = .success_zir;
             log.debug("AstGen cached success: {s}", .{file.sub_file_path});
 
@@ -2724,9 +2735,9 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
         },
         .parse_failure, .astgen_failure, .success_zir => {
             const unchanged_metadata =
-                stat.size == file.stat_size and
-                stat.mtime == file.stat_mtime and
-                stat.inode == file.stat_inode;
+                stat.size == file.stat.size and
+                stat.mtime == file.stat.mtime and
+                stat.inode == file.stat.inode;
 
             if (unchanged_metadata) {
                 log.debug("unmodified metadata of file: {s}", .{file.sub_file_path});
@@ -2787,9 +2798,11 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
     if (amt != stat.size)
         return error.UnexpectedEndOfFile;
 
-    file.stat_size = stat.size;
-    file.stat_inode = stat.inode;
-    file.stat_mtime = stat.mtime;
+    file.stat = .{
+        .size = stat.size,
+        .inode = stat.inode,
+        .mtime = stat.mtime,
+    };
     file.source = source;
     file.source_loaded = true;
 
@@ -3069,9 +3082,11 @@ pub fn populateBuiltinFile(mod: *Module) !void {
 
             try writeBuiltinFile(file, builtin_pkg);
         } else {
-            file.stat_size = stat.size;
-            file.stat_inode = stat.inode;
-            file.stat_mtime = stat.mtime;
+            file.stat = .{
+                .size = stat.size,
+                .inode = stat.inode,
+                .mtime = stat.mtime,
+            };
         }
     } else |err| switch (err) {
         error.BadPathName => unreachable, // it's always "builtin.zig"
@@ -3099,9 +3114,11 @@ pub fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void {
     try af.file.writeAll(file.source);
     try af.finish();
 
-    file.stat_size = file.source.len;
-    file.stat_inode = 0; // dummy value
-    file.stat_mtime = 0; // dummy value
+    file.stat = .{
+        .size = file.source.len,
+        .inode = 0, // dummy value
+        .mtime = 0, // dummy value
+    };
 }
 
 pub fn mapOldZirToNew(
@@ -3382,16 +3399,16 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void {
         }
 
         if (mod.comp.whole_cache_manifest) |man| {
-            assert(file.source_loaded);
+            const source = file.getSource(gpa) catch |err| {
+                try reportRetryableFileError(mod, file, "unable to load source: {s}", .{@errorName(err)});
+                return error.AnalysisFail;
+            };
             const resolved_path = try file.pkg.root_src_directory.join(gpa, &.{
                 file.sub_file_path,
             });
             errdefer gpa.free(resolved_path);
-            try man.addFilePostContents(resolved_path, file.source, .{
-                .size = file.stat_size,
-                .inode = file.stat_inode,
-                .mtime = file.stat_mtime,
-            });
+
+            try man.addFilePostContents(resolved_path, source.bytes, source.stat);
         }
     } else {
         new_decl.analysis = .file_failure;
@@ -3723,9 +3740,7 @@ pub fn importPkg(mod: *Module, pkg: *Package) !ImportFileResult {
         .source_loaded = false,
         .tree_loaded = false,
         .zir_loaded = false,
-        .stat_size = undefined,
-        .stat_inode = undefined,
-        .stat_mtime = undefined,
+        .stat = undefined,
         .tree = undefined,
         .zir = undefined,
         .status = .never_loaded,
@@ -3793,9 +3808,7 @@ pub fn importFile(
         .source_loaded = false,
         .tree_loaded = false,
         .zir_loaded = false,
-        .stat_size = undefined,
-        .stat_inode = undefined,
-        .stat_mtime = undefined,
+        .stat = undefined,
         .tree = undefined,
         .zir = undefined,
         .status = .never_loaded,
@@ -3840,8 +3853,13 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
     var file = try cur_file.pkg.root_src_directory.handle.openFile(sub_file_path, .{});
     defer file.close();
 
-    const stat = try file.stat();
-    const size_usize = try std.math.cast(usize, stat.size);
+    const actual_stat = try file.stat();
+    const stat: Cache.File.Stat = .{
+        .size = actual_stat.size,
+        .inode = actual_stat.inode,
+        .mtime = actual_stat.mtime,
+    };
+    const size_usize = try std.math.cast(usize, actual_stat.size);
     const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), size_usize, 1, 0);
     errdefer gpa.free(bytes);
 
@@ -3852,11 +3870,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
     if (mod.comp.whole_cache_manifest) |man| {
         const copied_resolved_path = try gpa.dupe(u8, resolved_path);
         errdefer gpa.free(copied_resolved_path);
-        try man.addFilePostContents(copied_resolved_path, bytes, .{
-            .size = stat.size,
-            .inode = stat.inode,
-            .mtime = stat.mtime,
-        });
+        try man.addFilePostContents(copied_resolved_path, bytes, stat);
     }
 
     keep_resolved_path = true; // It's now owned by embed_table.
@@ -3864,9 +3878,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
     new_file.* = .{
         .sub_file_path = sub_file_path,
         .bytes = bytes,
-        .stat_size = stat.size,
-        .stat_inode = stat.inode,
-        .stat_mtime = stat.mtime,
+        .stat = stat,
         .pkg = cur_file.pkg,
         .owner_decl = undefined, // Set by Sema immediately after this function returns.
     };
@@ -3880,9 +3892,9 @@ pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void {
     const stat = try file.stat();
 
     const unchanged_metadata =
-        stat.size == embed_file.stat_size and
-        stat.mtime == embed_file.stat_mtime and
-        stat.inode == embed_file.stat_inode;
+        stat.size == embed_file.stat.size and
+        stat.mtime == embed_file.stat.mtime and
+        stat.inode == embed_file.stat.inode;
 
     if (unchanged_metadata) return;
 
@@ -3891,9 +3903,11 @@ pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void {
     const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), size_usize, 1, 0);
     gpa.free(embed_file.bytes);
     embed_file.bytes = bytes;
-    embed_file.stat_size = stat.size;
-    embed_file.stat_mtime = stat.mtime;
-    embed_file.stat_inode = stat.inode;
+    embed_file.stat = .{
+        .size = stat.size,
+        .mtime = stat.mtime,
+        .inode = stat.inode,
+    };
 
     mod.comp.mutex.lock();
     defer mod.comp.mutex.unlock();
@@ -5024,3 +5038,35 @@ pub fn linkerUpdateDecl(mod: *Module, decl: *Decl) !void {
         },
     };
 }
+
+fn reportRetryableFileError(
+    mod: *Module,
+    file: *File,
+    comptime format: []const u8,
+    args: anytype,
+) error{OutOfMemory}!void {
+    file.status = .retryable_failure;
+
+    const err_msg = try ErrorMsg.create(
+        mod.gpa,
+        .{
+            .file_scope = file,
+            .parent_decl_node = 0,
+            .lazy = .entire_file,
+        },
+        format,
+        args,
+    );
+    errdefer err_msg.destroy(mod.gpa);
+
+    mod.comp.mutex.lock();
+    defer mod.comp.mutex.unlock();
+
+    const gop = try mod.failed_files.getOrPut(mod.gpa, file);
+    if (gop.found_existing) {
+        if (gop.value_ptr.*) |old_err_msg| {
+            old_err_msg.destroy(mod.gpa);
+        }
+    }
+    gop.value_ptr.* = err_msg;
+}
diff --git a/src/main.zig b/src/main.zig
index 6f55e46eb1c2..4747772b8a88 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -3664,9 +3664,7 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
                 .zir_loaded = false,
                 .sub_file_path = "<stdin>",
                 .source = source_code,
-                .stat_size = undefined,
-                .stat_inode = undefined,
-                .stat_mtime = undefined,
+                .stat = undefined,
                 .tree = tree,
                 .tree_loaded = true,
                 .zir = undefined,
@@ -3860,9 +3858,11 @@ fn fmtPathFile(
             .zir_loaded = false,
             .sub_file_path = file_path,
             .source = source_code,
-            .stat_size = stat.size,
-            .stat_inode = stat.inode,
-            .stat_mtime = stat.mtime,
+            .stat = .{
+                .size = stat.size,
+                .inode = stat.inode,
+                .mtime = stat.mtime,
+            },
             .tree = tree,
             .tree_loaded = true,
             .zir = undefined,
@@ -4458,9 +4458,7 @@ pub fn cmdAstCheck(
         .zir_loaded = false,
         .sub_file_path = undefined,
         .source = undefined,
-        .stat_size = undefined,
-        .stat_inode = undefined,
-        .stat_mtime = undefined,
+        .stat = undefined,
         .tree = undefined,
         .zir = undefined,
         .pkg = undefined,
@@ -4485,9 +4483,11 @@ pub fn cmdAstCheck(
         file.sub_file_path = file_name;
         file.source = source;
         file.source_loaded = true;
-        file.stat_size = stat.size;
-        file.stat_inode = stat.inode;
-        file.stat_mtime = stat.mtime;
+        file.stat = .{
+            .size = stat.size,
+            .inode = stat.inode,
+            .mtime = stat.mtime,
+        };
     } else {
         const stdin = io.getStdIn();
         const source = readSourceFileToEndAlloc(arena, &stdin, null) catch |err| {
@@ -4496,7 +4496,7 @@ pub fn cmdAstCheck(
         file.sub_file_path = "<stdin>";
         file.source = source;
         file.source_loaded = true;
-        file.stat_size = source.len;
+        file.stat.size = source.len;
     }
 
     file.pkg = try Package.create(gpa, null, file.sub_file_path);
@@ -4609,9 +4609,11 @@ pub fn cmdChangelist(
         .zir_loaded = false,
         .sub_file_path = old_source_file,
         .source = undefined,
-        .stat_size = stat.size,
-        .stat_inode = stat.inode,
-        .stat_mtime = stat.mtime,
+        .stat = .{
+            .size = stat.size,
+            .inode = stat.inode,
+            .mtime = stat.mtime,
+        },
         .tree = undefined,
         .zir = undefined,
         .pkg = undefined,

From 208a6c7d6a58c1fc46ed832acaa8a882f1c6a1dd Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 21:01:20 -0700
Subject: [PATCH 07/17] stage2: fix not calling deinit() on
 whole_cache_manifest

Need to mark it as "needing cleanup" a bit earlier in the function.
---
 src/Compilation.zig | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index e71c84af3377..9361159c4ec0 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1926,6 +1926,7 @@ pub fn update(comp: *Compilation) !void {
         // We are about to obtain this lock, so here we give other processes a chance first.
         comp.bin_file.releaseLock();
 
+        comp.whole_cache_manifest = &man;
         man = comp.cache_parent.obtain();
         try comp.addNonIncrementalStuffToCacheManifest(&man);
 
@@ -1960,8 +1961,6 @@ pub fn update(comp: *Compilation) !void {
         }
         log.debug("CacheMode.whole cache miss for {s}", .{comp.bin_file.options.root_name});
 
-        comp.whole_cache_manifest = &man;
-
         // Initialize `bin_file.emit` with a temporary Directory so that compilation can
         // continue on the same path as incremental, using the temporary Directory.
         tmp_artifact_directory = d: {

From 73ba6bf30be02d65e19304b0ec45c9a2a495698e Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 21:28:56 -0700
Subject: [PATCH 08/17] stage2: fix memory leak of emit_bin.sub_path

Instead of juggling GPA-allocated sub_path (and ultimately dropping the
ball, in this analogy), `Compilation.create` allocates an
already-exactly-correct size `sub_path` that has the digest unpopulated.

This is then overwritten in place as necessary and used as the
`emit_bin.sub_path` value, and no allocations/frees are performed for
this file path.
---
 src/Compilation.zig | 60 +++++++++++++++++++++------------------------
 1 file changed, 28 insertions(+), 32 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 9361159c4ec0..4c31f8a9987e 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -98,9 +98,11 @@ clang_argv: []const []const u8,
 cache_parent: *Cache,
 /// Path to own executable for invoking `zig clang`.
 self_exe_path: ?[]const u8,
-/// null means -fno-emit-bin. Contains the basename of the
-/// outputted binary file in case we don't know the directory yet.
-whole_bin_basename: ?[]const u8,
+/// null means -fno-emit-bin.
+/// This is mutable memory allocated into the Compilation-lifetime arena (`arena_state`)
+/// of exactly the correct size for "o/[digest]/[basename]".
+/// The basename is of the outputted binary file in case we don't know the directory yet.
+whole_bin_sub_path: ?[]u8,
 zig_lib_directory: Directory,
 local_cache_directory: Directory,
 global_cache_directory: Directory,
@@ -1487,9 +1489,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         // can use it for communicating the result directory via `bin_file.emit`.
         // This is used to distinguish between -fno-emit-bin and -femit-bin
         // for `CacheMode.whole`.
-        const whole_bin_basename: ?[]const u8 = if (options.emit_bin) |x|
+        // This memory will be overwritten with the real digest in update() but
+        // the basename will be preserved.
+        const whole_bin_sub_path: ?[]u8 = if (options.emit_bin) |x|
             if (x.directory == null)
-                x.basename
+                try std.fmt.allocPrint(arena, "o" ++ std.fs.path.sep_str ++
+                    ("x" ** Cache.hex_digest_len) ++ std.fs.path.sep_str ++ "{s}", .{x.basename})
             else
                 null
         else
@@ -1600,7 +1605,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             .local_cache_directory = options.local_cache_directory,
             .global_cache_directory = options.global_cache_directory,
             .bin_file = bin_file,
-            .whole_bin_basename = whole_bin_basename,
+            .whole_bin_sub_path = whole_bin_sub_path,
             .emit_asm = options.emit_asm,
             .emit_llvm_ir = options.emit_llvm_ir,
             .emit_llvm_bc = options.emit_llvm_bc,
@@ -1665,7 +1670,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});
     }
 
-    const have_bin_emit = comp.bin_file.options.emit != null or comp.whole_bin_basename != null;
+    const have_bin_emit = comp.bin_file.options.emit != null or comp.whole_bin_sub_path != null;
 
     if (have_bin_emit and !comp.bin_file.options.skip_linker_dependencies) {
         // If we need to build glibc for the target, add work items for it.
@@ -1941,19 +1946,7 @@ pub fn update(comp: *Compilation) !void {
             log.debug("CacheMode.whole cache hit for {s}", .{comp.bin_file.options.root_name});
             const digest = man.final();
 
-            // Communicate the output binary location to parent Compilations.
-            if (comp.whole_bin_basename) |basename| {
-                const new_sub_path = try std.fs.path.join(comp.gpa, &.{
-                    "o", &digest, basename,
-                });
-                if (comp.bin_file.options.emit) |emit| {
-                    comp.gpa.free(emit.sub_path);
-                }
-                comp.bin_file.options.emit = .{
-                    .directory = comp.local_cache_directory,
-                    .sub_path = new_sub_path,
-                };
-            }
+            comp.wholeCacheModeSetBinFilePath(&digest);
 
             assert(comp.bin_file.lock == null);
             comp.bin_file.lock = man.toOwnedLock();
@@ -1989,13 +1982,10 @@ pub fn update(comp: *Compilation) !void {
         // This resets the link.File to operate as if we called openPath() in create()
         // instead of simulating -fno-emit-bin.
         var options = comp.bin_file.options;
-        if (comp.whole_bin_basename) |basename| {
-            if (options.emit) |emit| {
-                comp.gpa.free(emit.sub_path);
-            }
+        if (comp.whole_bin_sub_path) |sub_path| {
             options.emit = .{
                 .directory = tmp_artifact_directory.?,
-                .sub_path = basename,
+                .sub_path = std.fs.path.basename(sub_path),
             };
         }
         comp.bin_file.destroy();
@@ -2149,13 +2139,7 @@ pub fn update(comp: *Compilation) !void {
             log.warn("failed to write cache manifest: {s}", .{@errorName(err)});
         };
 
-        // Communicate the output binary location to parent Compilations.
-        if (comp.whole_bin_basename) |basename| {
-            comp.bin_file.options.emit = .{
-                .directory = comp.local_cache_directory,
-                .sub_path = try std.fs.path.join(comp.gpa, &.{ "o", &digest, basename }),
-            };
-        }
+        comp.wholeCacheModeSetBinFilePath(&digest);
 
         assert(comp.bin_file.lock == null);
         comp.bin_file.lock = man.toOwnedLock();
@@ -2163,6 +2147,18 @@ pub fn update(comp: *Compilation) !void {
     }
 }
 
+/// Communicate the output binary location to parent Compilations.
+fn wholeCacheModeSetBinFilePath(comp: *Compilation, digest: *const [Cache.hex_digest_len]u8) void {
+    const sub_path = comp.whole_bin_sub_path orelse return;
+    const digest_start = 2; // "o/[digest]/[basename]"
+    mem.copy(u8, sub_path[digest_start..], digest);
+
+    comp.bin_file.options.emit = .{
+        .directory = comp.local_cache_directory,
+        .sub_path = sub_path,
+    };
+}
+
 /// This is only observed at compile-time and used to emit a compile error
 /// to remind the programmer to update multiple related pieces of code that
 /// are in different locations. Bump this number when adding or deleting

From 013c286e432eacbd55a9f842cd277be11f72cc49 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 21:35:24 -0700
Subject: [PATCH 09/17] stage2: fix memory leak in
 addNonIncrementalStuffToCacheManifest

I accidentally used errdefer instead of defer for the function-local
arena.
---
 src/Compilation.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 4c31f8a9987e..c17b187f4f1c 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2170,7 +2170,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
     const target = comp.getTarget();
 
     var arena_allocator = std.heap.ArenaAllocator.init(gpa);
-    errdefer arena_allocator.deinit();
+    defer arena_allocator.deinit();
     const arena = arena_allocator.allocator();
 
     comptime assert(link_hash_implementation_version == 1);

From 55243cfb31e40f09503fbc5b4bb63e7a5d3d87eb Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 22:04:18 -0700
Subject: [PATCH 10/17] stage2: fix implibs

Broken by the introduction of `CacheMode.whole`, they are now working
again by using the same mechanism as emitted binary files.
---
 src/Compilation.zig | 57 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 14 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index c17b187f4f1c..5b14eacc06d0 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -103,6 +103,8 @@ self_exe_path: ?[]const u8,
 /// of exactly the correct size for "o/[digest]/[basename]".
 /// The basename is of the outputted binary file in case we don't know the directory yet.
 whole_bin_sub_path: ?[]u8,
+/// Same as `whole_bin_sub_path` but for implibs.
+whole_implib_sub_path: ?[]u8,
 zig_lib_directory: Directory,
 local_cache_directory: Directory,
 global_cache_directory: Directory,
@@ -1477,6 +1479,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 };
             }
 
+            // This is here for the same reason as in `bin_file_emit` above.
+            switch (cache_mode) {
+                .whole => break :blk null,
+                .incremental => {},
+            }
+
             // Use the same directory as the bin. The CLI already emits an
             // error if -fno-emit-bin is combined with -femit-implib.
             break :blk link.Emit{
@@ -1491,14 +1499,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         // for `CacheMode.whole`.
         // This memory will be overwritten with the real digest in update() but
         // the basename will be preserved.
-        const whole_bin_sub_path: ?[]u8 = if (options.emit_bin) |x|
-            if (x.directory == null)
-                try std.fmt.allocPrint(arena, "o" ++ std.fs.path.sep_str ++
-                    ("x" ** Cache.hex_digest_len) ++ std.fs.path.sep_str ++ "{s}", .{x.basename})
-            else
-                null
-        else
-            null;
+        const whole_bin_sub_path: ?[]u8 = try prepareWholeEmitSubPath(arena, options.emit_bin);
+        // Same thing but for implibs.
+        const whole_implib_sub_path: ?[]u8 = try prepareWholeEmitSubPath(arena, options.emit_implib);
 
         var system_libs: std.StringArrayHashMapUnmanaged(SystemLib) = .{};
         errdefer system_libs.deinit(gpa);
@@ -1606,6 +1609,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             .global_cache_directory = options.global_cache_directory,
             .bin_file = bin_file,
             .whole_bin_sub_path = whole_bin_sub_path,
+            .whole_implib_sub_path = whole_implib_sub_path,
             .emit_asm = options.emit_asm,
             .emit_llvm_ir = options.emit_llvm_ir,
             .emit_llvm_bc = options.emit_llvm_bc,
@@ -1988,6 +1992,12 @@ pub fn update(comp: *Compilation) !void {
                 .sub_path = std.fs.path.basename(sub_path),
             };
         }
+        if (comp.whole_implib_sub_path) |sub_path| {
+            options.implib_emit = .{
+                .directory = tmp_artifact_directory.?,
+                .sub_path = std.fs.path.basename(sub_path),
+            };
+        }
         comp.bin_file.destroy();
         comp.bin_file = try link.File.openPath(comp.gpa, options);
     }
@@ -2149,14 +2159,33 @@ pub fn update(comp: *Compilation) !void {
 
 /// Communicate the output binary location to parent Compilations.
 fn wholeCacheModeSetBinFilePath(comp: *Compilation, digest: *const [Cache.hex_digest_len]u8) void {
-    const sub_path = comp.whole_bin_sub_path orelse return;
     const digest_start = 2; // "o/[digest]/[basename]"
-    mem.copy(u8, sub_path[digest_start..], digest);
 
-    comp.bin_file.options.emit = .{
-        .directory = comp.local_cache_directory,
-        .sub_path = sub_path,
-    };
+    if (comp.whole_bin_sub_path) |sub_path| {
+        mem.copy(u8, sub_path[digest_start..], digest);
+
+        comp.bin_file.options.emit = .{
+            .directory = comp.local_cache_directory,
+            .sub_path = sub_path,
+        };
+    }
+
+    if (comp.whole_implib_sub_path) |sub_path| {
+        mem.copy(u8, sub_path[digest_start..], digest);
+
+        comp.bin_file.options.implib_emit = .{
+            .directory = comp.local_cache_directory,
+            .sub_path = sub_path,
+        };
+    }
+}
+
+fn prepareWholeEmitSubPath(arena: Allocator, opt_emit: ?EmitLoc) error{OutOfMemory}!?[]u8 {
+    const emit = opt_emit orelse return null;
+    if (emit.directory != null) return null;
+    const s = std.fs.path.sep_str;
+    const format = "o" ++ s ++ ("x" ** Cache.hex_digest_len) ++ s ++ "{s}";
+    return try std.fmt.allocPrint(arena, format, .{emit.basename});
 }
 
 /// This is only observed at compile-time and used to emit a compile error

From 9dc25dd0b695c907e861bfd5a974cf454e657228 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 30 Dec 2021 23:14:34 -0700
Subject: [PATCH 11/17] stage2: fix UAF of system_libs

---
 src/Compilation.zig | 2 +-
 src/link.zig        | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 5b14eacc06d0..8b761cd45032 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1985,7 +1985,7 @@ pub fn update(comp: *Compilation) !void {
 
         // This resets the link.File to operate as if we called openPath() in create()
         // instead of simulating -fno-emit-bin.
-        var options = comp.bin_file.options;
+        var options = comp.bin_file.options.move();
         if (comp.whole_bin_sub_path) |sub_path| {
             options.emit = .{
                 .directory = tmp_artifact_directory.?,
diff --git a/src/link.zig b/src/link.zig
index 215d24adca53..a720bdada2b9 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -167,6 +167,12 @@ pub const Options = struct {
     pub fn effectiveOutputMode(options: Options) std.builtin.OutputMode {
         return if (options.use_lld) .Obj else options.output_mode;
     }
+
+    pub fn move(self: *Options) Options {
+        const copied_state = self.*;
+        self.system_libs = .{};
+        return copied_state;
+    }
 };
 
 pub const File = struct {

From 6b14c58f63a177c26ed7ac51c25738ecb115462d Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sat, 1 Jan 2022 19:49:34 -0700
Subject: [PATCH 12/17] compiler-rt: simplify implementations

This improves readability as well as compatibility with stage2. Most of
compiler-rt is now enabled for stage2 with just a few functions disabled
(until stage2 passes more behavior tests).
---
 lib/std/special/compiler_rt.zig             | 984 ++++++++++----------
 lib/std/special/compiler_rt/absv.zig        |  53 +-
 lib/std/special/compiler_rt/atomics.zig     | 454 +++++----
 lib/std/special/compiler_rt/bswap.zig       | 110 +--
 lib/std/special/compiler_rt/cmp.zig         |  56 +-
 lib/std/special/compiler_rt/count0bits.zig  | 238 ++---
 lib/std/special/compiler_rt/fixuint.zig     |   2 +-
 lib/std/special/compiler_rt/floatXisf.zig   |  11 +-
 lib/std/special/compiler_rt/floatsiXf.zig   |  12 +-
 lib/std/special/compiler_rt/floatundisf.zig |   9 +-
 lib/std/special/compiler_rt/floatunsidf.zig |   9 +-
 lib/std/special/compiler_rt/floatunsisf.zig |   9 +-
 lib/std/special/compiler_rt/negXi2.zig      |  24 +-
 lib/std/special/compiler_rt/negv.zig        |  43 +-
 lib/std/special/compiler_rt/parity.zig      |  48 +-
 lib/std/special/compiler_rt/popcount.zig    |  52 +-
 lib/std/special/compiler_rt/shift.zig       |  24 +-
 src/type.zig                                |  13 +-
 18 files changed, 1152 insertions(+), 999 deletions(-)

diff --git a/lib/std/special/compiler_rt.zig b/lib/std/special/compiler_rt.zig
index 297fd1cb39d7..29c6bdc7c353 100644
--- a/lib/std/special/compiler_rt.zig
+++ b/lib/std/special/compiler_rt.zig
@@ -22,6 +22,12 @@ else
 const long_double_is_f128 = builtin.target.longDoubleIsF128();
 
 comptime {
+    // These files do their own comptime exporting logic.
+    if (!builtin.zig_is_stage2) {
+        _ = @import("compiler_rt/atomics.zig");
+    }
+    _ = @import("compiler_rt/clear_cache.zig").clear_cache;
+
     const __extenddftf2 = @import("compiler_rt/extendXfYf2.zig").__extenddftf2;
     @export(__extenddftf2, .{ .name = "__extenddftf2", .linkage = linkage });
     const __extendsftf2 = @import("compiler_rt/extendXfYf2.zig").__extendsftf2;
@@ -171,16 +177,16 @@ comptime {
     const __truncdfsf2 = @import("compiler_rt/truncXfYf2.zig").__truncdfsf2;
     @export(__truncdfsf2, .{ .name = "__truncdfsf2", .linkage = linkage });
 
-    if (!builtin.zig_is_stage2) {
-        if (!long_double_is_f128) {
-            // TODO implement these
-            //const __extendxftf2 = @import("compiler_rt/extendXfYf2.zig").__extendxftf2;
-            //@export(__extendxftf2, .{ .name = "__extendxftf2", .linkage = linkage });
+    if (!long_double_is_f128) {
+        // TODO implement these
+        //const __extendxftf2 = @import("compiler_rt/extendXfYf2.zig").__extendxftf2;
+        //@export(__extendxftf2, .{ .name = "__extendxftf2", .linkage = linkage });
 
-            //const __trunctfxf2 = @import("compiler_rt/truncXfYf2.zig").__trunctfxf2;
-            //@export(__trunctfxf2, .{ .name = "__trunctfxf2", .linkage = linkage });
-        }
+        //const __trunctfxf2 = @import("compiler_rt/truncXfYf2.zig").__trunctfxf2;
+        //@export(__trunctfxf2, .{ .name = "__trunctfxf2", .linkage = linkage });
+    }
 
+    if (!builtin.zig_is_stage2) {
         switch (arch) {
             .i386,
             .x86_64,
@@ -193,30 +199,29 @@ comptime {
             },
             else => {},
         }
+    }
 
-        // __clear_cache manages its own logic about whether to be exported or not.
-        _ = @import("compiler_rt/clear_cache.zig").clear_cache;
-
-        const __unordsf2 = @import("compiler_rt/compareXf2.zig").__unordsf2;
-        @export(__unordsf2, .{ .name = "__unordsf2", .linkage = linkage });
-        const __unorddf2 = @import("compiler_rt/compareXf2.zig").__unorddf2;
-        @export(__unorddf2, .{ .name = "__unorddf2", .linkage = linkage });
-        const __unordtf2 = @import("compiler_rt/compareXf2.zig").__unordtf2;
-        @export(__unordtf2, .{ .name = "__unordtf2", .linkage = linkage });
-
-        const __addsf3 = @import("compiler_rt/addXf3.zig").__addsf3;
-        @export(__addsf3, .{ .name = "__addsf3", .linkage = linkage });
-        const __adddf3 = @import("compiler_rt/addXf3.zig").__adddf3;
-        @export(__adddf3, .{ .name = "__adddf3", .linkage = linkage });
-        const __addtf3 = @import("compiler_rt/addXf3.zig").__addtf3;
-        @export(__addtf3, .{ .name = "__addtf3", .linkage = linkage });
-        const __subsf3 = @import("compiler_rt/addXf3.zig").__subsf3;
-        @export(__subsf3, .{ .name = "__subsf3", .linkage = linkage });
-        const __subdf3 = @import("compiler_rt/addXf3.zig").__subdf3;
-        @export(__subdf3, .{ .name = "__subdf3", .linkage = linkage });
-        const __subtf3 = @import("compiler_rt/addXf3.zig").__subtf3;
-        @export(__subtf3, .{ .name = "__subtf3", .linkage = linkage });
+    const __unordsf2 = @import("compiler_rt/compareXf2.zig").__unordsf2;
+    @export(__unordsf2, .{ .name = "__unordsf2", .linkage = linkage });
+    const __unorddf2 = @import("compiler_rt/compareXf2.zig").__unorddf2;
+    @export(__unorddf2, .{ .name = "__unorddf2", .linkage = linkage });
+    const __unordtf2 = @import("compiler_rt/compareXf2.zig").__unordtf2;
+    @export(__unordtf2, .{ .name = "__unordtf2", .linkage = linkage });
+
+    const __addsf3 = @import("compiler_rt/addXf3.zig").__addsf3;
+    @export(__addsf3, .{ .name = "__addsf3", .linkage = linkage });
+    const __adddf3 = @import("compiler_rt/addXf3.zig").__adddf3;
+    @export(__adddf3, .{ .name = "__adddf3", .linkage = linkage });
+    const __addtf3 = @import("compiler_rt/addXf3.zig").__addtf3;
+    @export(__addtf3, .{ .name = "__addtf3", .linkage = linkage });
+    const __subsf3 = @import("compiler_rt/addXf3.zig").__subsf3;
+    @export(__subsf3, .{ .name = "__subsf3", .linkage = linkage });
+    const __subdf3 = @import("compiler_rt/addXf3.zig").__subdf3;
+    @export(__subdf3, .{ .name = "__subdf3", .linkage = linkage });
+    const __subtf3 = @import("compiler_rt/addXf3.zig").__subtf3;
+    @export(__subtf3, .{ .name = "__subtf3", .linkage = linkage });
 
+    if (!builtin.zig_is_stage2) {
         const __mulsf3 = @import("compiler_rt/mulXf3.zig").__mulsf3;
         @export(__mulsf3, .{ .name = "__mulsf3", .linkage = linkage });
         const __muldf3 = @import("compiler_rt/mulXf3.zig").__muldf3;
@@ -230,21 +235,23 @@ comptime {
         @export(__divdf3, .{ .name = "__divdf3", .linkage = linkage });
         const __divtf3 = @import("compiler_rt/divtf3.zig").__divtf3;
         @export(__divtf3, .{ .name = "__divtf3", .linkage = linkage });
+    }
 
-        // Integral bit manipulation
-        const __ashldi3 = @import("compiler_rt/shift.zig").__ashldi3;
-        @export(__ashldi3, .{ .name = "__ashldi3", .linkage = linkage });
-        const __ashlti3 = @import("compiler_rt/shift.zig").__ashlti3;
-        @export(__ashlti3, .{ .name = "__ashlti3", .linkage = linkage });
-        const __ashrdi3 = @import("compiler_rt/shift.zig").__ashrdi3;
-        @export(__ashrdi3, .{ .name = "__ashrdi3", .linkage = linkage });
-        const __ashrti3 = @import("compiler_rt/shift.zig").__ashrti3;
-        @export(__ashrti3, .{ .name = "__ashrti3", .linkage = linkage });
-        const __lshrdi3 = @import("compiler_rt/shift.zig").__lshrdi3;
-        @export(__lshrdi3, .{ .name = "__lshrdi3", .linkage = linkage });
-        const __lshrti3 = @import("compiler_rt/shift.zig").__lshrti3;
-        @export(__lshrti3, .{ .name = "__lshrti3", .linkage = linkage });
+    // Integral bit manipulation
+    const __ashldi3 = @import("compiler_rt/shift.zig").__ashldi3;
+    @export(__ashldi3, .{ .name = "__ashldi3", .linkage = linkage });
+    const __ashlti3 = @import("compiler_rt/shift.zig").__ashlti3;
+    @export(__ashlti3, .{ .name = "__ashlti3", .linkage = linkage });
+    const __ashrdi3 = @import("compiler_rt/shift.zig").__ashrdi3;
+    @export(__ashrdi3, .{ .name = "__ashrdi3", .linkage = linkage });
+    const __ashrti3 = @import("compiler_rt/shift.zig").__ashrti3;
+    @export(__ashrti3, .{ .name = "__ashrti3", .linkage = linkage });
+    const __lshrdi3 = @import("compiler_rt/shift.zig").__lshrdi3;
+    @export(__lshrdi3, .{ .name = "__lshrdi3", .linkage = linkage });
+    const __lshrti3 = @import("compiler_rt/shift.zig").__lshrti3;
+    @export(__lshrti3, .{ .name = "__lshrti3", .linkage = linkage });
 
+    if (!builtin.zig_is_stage2) {
         const __clzsi2 = @import("compiler_rt/count0bits.zig").__clzsi2;
         @export(__clzsi2, .{ .name = "__clzsi2", .linkage = linkage });
         const __clzdi2 = @import("compiler_rt/count0bits.zig").__clzdi2;
@@ -263,461 +270,466 @@ comptime {
         @export(__ffsdi2, .{ .name = "__ffsdi2", .linkage = linkage });
         const __ffsti2 = @import("compiler_rt/count0bits.zig").__ffsti2;
         @export(__ffsti2, .{ .name = "__ffsti2", .linkage = linkage });
+    }
 
-        const __paritysi2 = @import("compiler_rt/parity.zig").__paritysi2;
-        @export(__paritysi2, .{ .name = "__paritysi2", .linkage = linkage });
-        const __paritydi2 = @import("compiler_rt/parity.zig").__paritydi2;
-        @export(__paritydi2, .{ .name = "__paritydi2", .linkage = linkage });
-        const __parityti2 = @import("compiler_rt/parity.zig").__parityti2;
-        @export(__parityti2, .{ .name = "__parityti2", .linkage = linkage });
-        const __popcountsi2 = @import("compiler_rt/popcount.zig").__popcountsi2;
-        @export(__popcountsi2, .{ .name = "__popcountsi2", .linkage = linkage });
-        const __popcountdi2 = @import("compiler_rt/popcount.zig").__popcountdi2;
-        @export(__popcountdi2, .{ .name = "__popcountdi2", .linkage = linkage });
-        const __popcountti2 = @import("compiler_rt/popcount.zig").__popcountti2;
-        @export(__popcountti2, .{ .name = "__popcountti2", .linkage = linkage });
-        const __bswapsi2 = @import("compiler_rt/bswap.zig").__bswapsi2;
-        @export(__bswapsi2, .{ .name = "__bswapsi2", .linkage = linkage });
-        const __bswapdi2 = @import("compiler_rt/bswap.zig").__bswapdi2;
-        @export(__bswapdi2, .{ .name = "__bswapdi2", .linkage = linkage });
-        const __bswapti2 = @import("compiler_rt/bswap.zig").__bswapti2;
-        @export(__bswapti2, .{ .name = "__bswapti2", .linkage = linkage });
-
-        // Integral / floating point conversion (part 1/2)
-        const __floatsidf = @import("compiler_rt/floatsiXf.zig").__floatsidf;
-        @export(__floatsidf, .{ .name = "__floatsidf", .linkage = linkage });
-        const __floatsisf = @import("compiler_rt/floatsiXf.zig").__floatsisf;
-        @export(__floatsisf, .{ .name = "__floatsisf", .linkage = linkage });
-        const __floatdidf = @import("compiler_rt/floatdidf.zig").__floatdidf;
-        @export(__floatdidf, .{ .name = "__floatdidf", .linkage = linkage });
-        const __floatsitf = @import("compiler_rt/floatsiXf.zig").__floatsitf;
-        @export(__floatsitf, .{ .name = "__floatsitf", .linkage = linkage });
-
-        const __floatunsisf = @import("compiler_rt/floatunsisf.zig").__floatunsisf;
-        @export(__floatunsisf, .{ .name = "__floatunsisf", .linkage = linkage });
+    const __paritysi2 = @import("compiler_rt/parity.zig").__paritysi2;
+    @export(__paritysi2, .{ .name = "__paritysi2", .linkage = linkage });
+    const __paritydi2 = @import("compiler_rt/parity.zig").__paritydi2;
+    @export(__paritydi2, .{ .name = "__paritydi2", .linkage = linkage });
+    const __parityti2 = @import("compiler_rt/parity.zig").__parityti2;
+    @export(__parityti2, .{ .name = "__parityti2", .linkage = linkage });
+
+    const __popcountsi2 = @import("compiler_rt/popcount.zig").__popcountsi2;
+    @export(__popcountsi2, .{ .name = "__popcountsi2", .linkage = linkage });
+    const __popcountdi2 = @import("compiler_rt/popcount.zig").__popcountdi2;
+    @export(__popcountdi2, .{ .name = "__popcountdi2", .linkage = linkage });
+    const __popcountti2 = @import("compiler_rt/popcount.zig").__popcountti2;
+    @export(__popcountti2, .{ .name = "__popcountti2", .linkage = linkage });
+
+    const __bswapsi2 = @import("compiler_rt/bswap.zig").__bswapsi2;
+    @export(__bswapsi2, .{ .name = "__bswapsi2", .linkage = linkage });
+    const __bswapdi2 = @import("compiler_rt/bswap.zig").__bswapdi2;
+    @export(__bswapdi2, .{ .name = "__bswapdi2", .linkage = linkage });
+    const __bswapti2 = @import("compiler_rt/bswap.zig").__bswapti2;
+    @export(__bswapti2, .{ .name = "__bswapti2", .linkage = linkage });
+
+    // Integral / floating point conversion (part 1/2)
+    const __floatsidf = @import("compiler_rt/floatsiXf.zig").__floatsidf;
+    @export(__floatsidf, .{ .name = "__floatsidf", .linkage = linkage });
+    const __floatsisf = @import("compiler_rt/floatsiXf.zig").__floatsisf;
+    @export(__floatsisf, .{ .name = "__floatsisf", .linkage = linkage });
+    const __floatdidf = @import("compiler_rt/floatdidf.zig").__floatdidf;
+    @export(__floatdidf, .{ .name = "__floatdidf", .linkage = linkage });
+    const __floatsitf = @import("compiler_rt/floatsiXf.zig").__floatsitf;
+    @export(__floatsitf, .{ .name = "__floatsitf", .linkage = linkage });
+
+    const __floatunsisf = @import("compiler_rt/floatunsisf.zig").__floatunsisf;
+    @export(__floatunsisf, .{ .name = "__floatunsisf", .linkage = linkage });
+    if (!builtin.zig_is_stage2) {
         const __floatundisf = @import("compiler_rt/floatundisf.zig").__floatundisf;
         @export(__floatundisf, .{ .name = "__floatundisf", .linkage = linkage });
-        const __floatunsidf = @import("compiler_rt/floatunsidf.zig").__floatunsidf;
-        @export(__floatunsidf, .{ .name = "__floatunsidf", .linkage = linkage });
-        const __floatundidf = @import("compiler_rt/floatundidf.zig").__floatundidf;
-        @export(__floatundidf, .{ .name = "__floatundidf", .linkage = linkage });
-
-        const __floatditf = @import("compiler_rt/floatditf.zig").__floatditf;
-        @export(__floatditf, .{ .name = "__floatditf", .linkage = linkage });
-        const __floattitf = @import("compiler_rt/floattitf.zig").__floattitf;
-        @export(__floattitf, .{ .name = "__floattitf", .linkage = linkage });
-        const __floattidf = @import("compiler_rt/floattidf.zig").__floattidf;
-        @export(__floattidf, .{ .name = "__floattidf", .linkage = linkage });
-        const __floattisf = @import("compiler_rt/floatXisf.zig").__floattisf;
-        @export(__floattisf, .{ .name = "__floattisf", .linkage = linkage });
-        const __floatdisf = @import("compiler_rt/floatXisf.zig").__floatdisf;
-        @export(__floatdisf, .{ .name = "__floatdisf", .linkage = linkage });
-
-        const __floatunditf = @import("compiler_rt/floatunditf.zig").__floatunditf;
-        @export(__floatunditf, .{ .name = "__floatunditf", .linkage = linkage });
-        const __floatunsitf = @import("compiler_rt/floatunsitf.zig").__floatunsitf;
-        @export(__floatunsitf, .{ .name = "__floatunsitf", .linkage = linkage });
-
-        const __floatuntitf = @import("compiler_rt/floatuntitf.zig").__floatuntitf;
-        @export(__floatuntitf, .{ .name = "__floatuntitf", .linkage = linkage });
-        const __floatuntidf = @import("compiler_rt/floatuntidf.zig").__floatuntidf;
-        @export(__floatuntidf, .{ .name = "__floatuntidf", .linkage = linkage });
-        const __floatuntisf = @import("compiler_rt/floatuntisf.zig").__floatuntisf;
-        @export(__floatuntisf, .{ .name = "__floatuntisf", .linkage = linkage });
-
-        const __truncsfhf2 = @import("compiler_rt/truncXfYf2.zig").__truncsfhf2;
-        @export(__truncsfhf2, .{ .name = "__truncsfhf2", .linkage = linkage });
-        if (!is_test) {
-            @export(__truncsfhf2, .{ .name = "__gnu_f2h_ieee", .linkage = linkage });
-        }
-        const __extendsfdf2 = @import("compiler_rt/extendXfYf2.zig").__extendsfdf2;
-        @export(__extendsfdf2, .{ .name = "__extendsfdf2", .linkage = linkage });
-
-        // Integral / floating point conversion (part 2/2)
-        const __fixunssfsi = @import("compiler_rt/fixunssfsi.zig").__fixunssfsi;
-        @export(__fixunssfsi, .{ .name = "__fixunssfsi", .linkage = linkage });
-        const __fixunssfdi = @import("compiler_rt/fixunssfdi.zig").__fixunssfdi;
-        @export(__fixunssfdi, .{ .name = "__fixunssfdi", .linkage = linkage });
-        const __fixunssfti = @import("compiler_rt/fixunssfti.zig").__fixunssfti;
-        @export(__fixunssfti, .{ .name = "__fixunssfti", .linkage = linkage });
-
-        const __fixunsdfsi = @import("compiler_rt/fixunsdfsi.zig").__fixunsdfsi;
-        @export(__fixunsdfsi, .{ .name = "__fixunsdfsi", .linkage = linkage });
-        const __fixunsdfdi = @import("compiler_rt/fixunsdfdi.zig").__fixunsdfdi;
-        @export(__fixunsdfdi, .{ .name = "__fixunsdfdi", .linkage = linkage });
-        const __fixunsdfti = @import("compiler_rt/fixunsdfti.zig").__fixunsdfti;
-        @export(__fixunsdfti, .{ .name = "__fixunsdfti", .linkage = linkage });
-
-        const __fixunstfsi = @import("compiler_rt/fixunstfsi.zig").__fixunstfsi;
-        @export(__fixunstfsi, .{ .name = "__fixunstfsi", .linkage = linkage });
-        const __fixunstfdi = @import("compiler_rt/fixunstfdi.zig").__fixunstfdi;
-        @export(__fixunstfdi, .{ .name = "__fixunstfdi", .linkage = linkage });
-        const __fixunstfti = @import("compiler_rt/fixunstfti.zig").__fixunstfti;
-        @export(__fixunstfti, .{ .name = "__fixunstfti", .linkage = linkage });
-
-        const __fixdfdi = @import("compiler_rt/fixdfdi.zig").__fixdfdi;
-        @export(__fixdfdi, .{ .name = "__fixdfdi", .linkage = linkage });
-        const __fixdfsi = @import("compiler_rt/fixdfsi.zig").__fixdfsi;
-        @export(__fixdfsi, .{ .name = "__fixdfsi", .linkage = linkage });
-        const __fixdfti = @import("compiler_rt/fixdfti.zig").__fixdfti;
-        @export(__fixdfti, .{ .name = "__fixdfti", .linkage = linkage });
-        const __fixsfdi = @import("compiler_rt/fixsfdi.zig").__fixsfdi;
-        @export(__fixsfdi, .{ .name = "__fixsfdi", .linkage = linkage });
-        const __fixsfsi = @import("compiler_rt/fixsfsi.zig").__fixsfsi;
-        @export(__fixsfsi, .{ .name = "__fixsfsi", .linkage = linkage });
-        const __fixsfti = @import("compiler_rt/fixsfti.zig").__fixsfti;
-        @export(__fixsfti, .{ .name = "__fixsfti", .linkage = linkage });
-        const __fixtfdi = @import("compiler_rt/fixtfdi.zig").__fixtfdi;
-        @export(__fixtfdi, .{ .name = "__fixtfdi", .linkage = linkage });
-        const __fixtfsi = @import("compiler_rt/fixtfsi.zig").__fixtfsi;
-        @export(__fixtfsi, .{ .name = "__fixtfsi", .linkage = linkage });
-        const __fixtfti = @import("compiler_rt/fixtfti.zig").__fixtfti;
-        @export(__fixtfti, .{ .name = "__fixtfti", .linkage = linkage });
-
-        const __udivmoddi4 = @import("compiler_rt/int.zig").__udivmoddi4;
-        @export(__udivmoddi4, .{ .name = "__udivmoddi4", .linkage = linkage });
-
-        if (is_darwin) {
-            const __isPlatformVersionAtLeast = @import("compiler_rt/os_version_check.zig").__isPlatformVersionAtLeast;
-            @export(__isPlatformVersionAtLeast, .{ .name = "__isPlatformVersionAtLeast", .linkage = linkage });
-        }
+    }
+    const __floatunsidf = @import("compiler_rt/floatunsidf.zig").__floatunsidf;
+    @export(__floatunsidf, .{ .name = "__floatunsidf", .linkage = linkage });
+    const __floatundidf = @import("compiler_rt/floatundidf.zig").__floatundidf;
+    @export(__floatundidf, .{ .name = "__floatundidf", .linkage = linkage });
+
+    const __floatditf = @import("compiler_rt/floatditf.zig").__floatditf;
+    @export(__floatditf, .{ .name = "__floatditf", .linkage = linkage });
+    const __floattitf = @import("compiler_rt/floattitf.zig").__floattitf;
+    @export(__floattitf, .{ .name = "__floattitf", .linkage = linkage });
+    const __floattidf = @import("compiler_rt/floattidf.zig").__floattidf;
+    @export(__floattidf, .{ .name = "__floattidf", .linkage = linkage });
+    const __floattisf = @import("compiler_rt/floatXisf.zig").__floattisf;
+    @export(__floattisf, .{ .name = "__floattisf", .linkage = linkage });
+    const __floatdisf = @import("compiler_rt/floatXisf.zig").__floatdisf;
+    @export(__floatdisf, .{ .name = "__floatdisf", .linkage = linkage });
+
+    const __floatunditf = @import("compiler_rt/floatunditf.zig").__floatunditf;
+    @export(__floatunditf, .{ .name = "__floatunditf", .linkage = linkage });
+    const __floatunsitf = @import("compiler_rt/floatunsitf.zig").__floatunsitf;
+    @export(__floatunsitf, .{ .name = "__floatunsitf", .linkage = linkage });
+
+    const __floatuntitf = @import("compiler_rt/floatuntitf.zig").__floatuntitf;
+    @export(__floatuntitf, .{ .name = "__floatuntitf", .linkage = linkage });
+    const __floatuntidf = @import("compiler_rt/floatuntidf.zig").__floatuntidf;
+    @export(__floatuntidf, .{ .name = "__floatuntidf", .linkage = linkage });
+    const __floatuntisf = @import("compiler_rt/floatuntisf.zig").__floatuntisf;
+    @export(__floatuntisf, .{ .name = "__floatuntisf", .linkage = linkage });
+
+    const __truncsfhf2 = @import("compiler_rt/truncXfYf2.zig").__truncsfhf2;
+    @export(__truncsfhf2, .{ .name = "__truncsfhf2", .linkage = linkage });
+    if (!is_test) {
+        @export(__truncsfhf2, .{ .name = "__gnu_f2h_ieee", .linkage = linkage });
+    }
+    const __extendsfdf2 = @import("compiler_rt/extendXfYf2.zig").__extendsfdf2;
+    @export(__extendsfdf2, .{ .name = "__extendsfdf2", .linkage = linkage });
+
+    // Integral / floating point conversion (part 2/2)
+    const __fixunssfsi = @import("compiler_rt/fixunssfsi.zig").__fixunssfsi;
+    @export(__fixunssfsi, .{ .name = "__fixunssfsi", .linkage = linkage });
+    const __fixunssfdi = @import("compiler_rt/fixunssfdi.zig").__fixunssfdi;
+    @export(__fixunssfdi, .{ .name = "__fixunssfdi", .linkage = linkage });
+    const __fixunssfti = @import("compiler_rt/fixunssfti.zig").__fixunssfti;
+    @export(__fixunssfti, .{ .name = "__fixunssfti", .linkage = linkage });
+
+    const __fixunsdfsi = @import("compiler_rt/fixunsdfsi.zig").__fixunsdfsi;
+    @export(__fixunsdfsi, .{ .name = "__fixunsdfsi", .linkage = linkage });
+    const __fixunsdfdi = @import("compiler_rt/fixunsdfdi.zig").__fixunsdfdi;
+    @export(__fixunsdfdi, .{ .name = "__fixunsdfdi", .linkage = linkage });
+    const __fixunsdfti = @import("compiler_rt/fixunsdfti.zig").__fixunsdfti;
+    @export(__fixunsdfti, .{ .name = "__fixunsdfti", .linkage = linkage });
+
+    const __fixunstfsi = @import("compiler_rt/fixunstfsi.zig").__fixunstfsi;
+    @export(__fixunstfsi, .{ .name = "__fixunstfsi", .linkage = linkage });
+    const __fixunstfdi = @import("compiler_rt/fixunstfdi.zig").__fixunstfdi;
+    @export(__fixunstfdi, .{ .name = "__fixunstfdi", .linkage = linkage });
+    const __fixunstfti = @import("compiler_rt/fixunstfti.zig").__fixunstfti;
+    @export(__fixunstfti, .{ .name = "__fixunstfti", .linkage = linkage });
+
+    const __fixdfdi = @import("compiler_rt/fixdfdi.zig").__fixdfdi;
+    @export(__fixdfdi, .{ .name = "__fixdfdi", .linkage = linkage });
+    const __fixdfsi = @import("compiler_rt/fixdfsi.zig").__fixdfsi;
+    @export(__fixdfsi, .{ .name = "__fixdfsi", .linkage = linkage });
+    const __fixdfti = @import("compiler_rt/fixdfti.zig").__fixdfti;
+    @export(__fixdfti, .{ .name = "__fixdfti", .linkage = linkage });
+    const __fixsfdi = @import("compiler_rt/fixsfdi.zig").__fixsfdi;
+    @export(__fixsfdi, .{ .name = "__fixsfdi", .linkage = linkage });
+    const __fixsfsi = @import("compiler_rt/fixsfsi.zig").__fixsfsi;
+    @export(__fixsfsi, .{ .name = "__fixsfsi", .linkage = linkage });
+    const __fixsfti = @import("compiler_rt/fixsfti.zig").__fixsfti;
+    @export(__fixsfti, .{ .name = "__fixsfti", .linkage = linkage });
+    const __fixtfdi = @import("compiler_rt/fixtfdi.zig").__fixtfdi;
+    @export(__fixtfdi, .{ .name = "__fixtfdi", .linkage = linkage });
+    const __fixtfsi = @import("compiler_rt/fixtfsi.zig").__fixtfsi;
+    @export(__fixtfsi, .{ .name = "__fixtfsi", .linkage = linkage });
+    const __fixtfti = @import("compiler_rt/fixtfti.zig").__fixtfti;
+    @export(__fixtfti, .{ .name = "__fixtfti", .linkage = linkage });
+
+    const __udivmoddi4 = @import("compiler_rt/int.zig").__udivmoddi4;
+    @export(__udivmoddi4, .{ .name = "__udivmoddi4", .linkage = linkage });
+
+    if (is_darwin) {
+        const __isPlatformVersionAtLeast = @import("compiler_rt/os_version_check.zig").__isPlatformVersionAtLeast;
+        @export(__isPlatformVersionAtLeast, .{ .name = "__isPlatformVersionAtLeast", .linkage = linkage });
+    }
 
-        // Integral arithmetic
-        const __negsi2 = @import("compiler_rt/negXi2.zig").__negsi2;
-        @export(__negsi2, .{ .name = "__negsi2", .linkage = linkage });
-        const __negdi2 = @import("compiler_rt/negXi2.zig").__negdi2;
-        @export(__negdi2, .{ .name = "__negdi2", .linkage = linkage });
-        const __negti2 = @import("compiler_rt/negXi2.zig").__negti2;
-        @export(__negti2, .{ .name = "__negti2", .linkage = linkage });
-        const __mulsi3 = @import("compiler_rt/int.zig").__mulsi3;
-        @export(__mulsi3, .{ .name = "__mulsi3", .linkage = linkage });
-        const __muldi3 = @import("compiler_rt/muldi3.zig").__muldi3;
-        @export(__muldi3, .{ .name = "__muldi3", .linkage = linkage });
-        const __divmoddi4 = @import("compiler_rt/int.zig").__divmoddi4;
-        @export(__divmoddi4, .{ .name = "__divmoddi4", .linkage = linkage });
-        const __divsi3 = @import("compiler_rt/int.zig").__divsi3;
-        @export(__divsi3, .{ .name = "__divsi3", .linkage = linkage });
-        const __divdi3 = @import("compiler_rt/int.zig").__divdi3;
-        @export(__divdi3, .{ .name = "__divdi3", .linkage = linkage });
-        const __udivsi3 = @import("compiler_rt/int.zig").__udivsi3;
-        @export(__udivsi3, .{ .name = "__udivsi3", .linkage = linkage });
-        const __udivdi3 = @import("compiler_rt/int.zig").__udivdi3;
-        @export(__udivdi3, .{ .name = "__udivdi3", .linkage = linkage });
-        const __modsi3 = @import("compiler_rt/int.zig").__modsi3;
-        @export(__modsi3, .{ .name = "__modsi3", .linkage = linkage });
-        const __moddi3 = @import("compiler_rt/int.zig").__moddi3;
-        @export(__moddi3, .{ .name = "__moddi3", .linkage = linkage });
-        const __umodsi3 = @import("compiler_rt/int.zig").__umodsi3;
-        @export(__umodsi3, .{ .name = "__umodsi3", .linkage = linkage });
-        const __umoddi3 = @import("compiler_rt/int.zig").__umoddi3;
-        @export(__umoddi3, .{ .name = "__umoddi3", .linkage = linkage });
-        const __divmodsi4 = @import("compiler_rt/int.zig").__divmodsi4;
-        @export(__divmodsi4, .{ .name = "__divmodsi4", .linkage = linkage });
-        const __udivmodsi4 = @import("compiler_rt/int.zig").__udivmodsi4;
-        @export(__udivmodsi4, .{ .name = "__udivmodsi4", .linkage = linkage });
-
-        // Integral arithmetic with trapping overflow
-        const __absvsi2 = @import("compiler_rt/absv.zig").__absvsi2;
-        @export(__absvsi2, .{ .name = "__absvsi2", .linkage = linkage });
-        const __absvdi2 = @import("compiler_rt/absv.zig").__absvdi2;
-        @export(__absvdi2, .{ .name = "__absvdi2", .linkage = linkage });
-        const __absvti2 = @import("compiler_rt/absv.zig").__absvti2;
-        @export(__absvti2, .{ .name = "__absvti2", .linkage = linkage });
-        const __negvsi2 = @import("compiler_rt/negv.zig").__negvsi2;
-        @export(__negvsi2, .{ .name = "__negvsi2", .linkage = linkage });
-        const __negvdi2 = @import("compiler_rt/negv.zig").__negvdi2;
-        @export(__negvdi2, .{ .name = "__negvdi2", .linkage = linkage });
-        const __negvti2 = @import("compiler_rt/negv.zig").__negvti2;
-        @export(__negvti2, .{ .name = "__negvti2", .linkage = linkage });
-
-        // missing: Integral arithmetic which returns if overflow
-
-        // Integral comparison
-        // (a <  b) => 0
-        // (a == b) => 1
-        // (a >  b) => 2
-        const __cmpsi2 = @import("compiler_rt/cmp.zig").__cmpsi2;
-        @export(__cmpsi2, .{ .name = "__cmpsi2", .linkage = linkage });
-        const __cmpdi2 = @import("compiler_rt/cmp.zig").__cmpdi2;
-        @export(__cmpdi2, .{ .name = "__cmpdi2", .linkage = linkage });
-        const __cmpti2 = @import("compiler_rt/cmp.zig").__cmpti2;
-        @export(__cmpti2, .{ .name = "__cmpti2", .linkage = linkage });
-        const __ucmpsi2 = @import("compiler_rt/cmp.zig").__ucmpsi2;
-        @export(__ucmpsi2, .{ .name = "__ucmpsi2", .linkage = linkage });
-        const __ucmpdi2 = @import("compiler_rt/cmp.zig").__ucmpdi2;
-        @export(__ucmpdi2, .{ .name = "__ucmpdi2", .linkage = linkage });
-        const __ucmpti2 = @import("compiler_rt/cmp.zig").__ucmpti2;
-        @export(__ucmpti2, .{ .name = "__ucmpti2", .linkage = linkage });
-
-        // missing: Floating point raised to integer power
-
-        // missing: Complex arithmetic
-        // (a + ib) * (c + id)
-        // (a + ib) / (c + id)
-
-        const __negsf2 = @import("compiler_rt/negXf2.zig").__negsf2;
-        @export(__negsf2, .{ .name = "__negsf2", .linkage = linkage });
-        const __negdf2 = @import("compiler_rt/negXf2.zig").__negdf2;
-        @export(__negdf2, .{ .name = "__negdf2", .linkage = linkage });
-
-        if (builtin.link_libc and os_tag == .openbsd) {
-            const __emutls_get_address = @import("compiler_rt/emutls.zig").__emutls_get_address;
-            @export(__emutls_get_address, .{ .name = "__emutls_get_address", .linkage = linkage });
-        }
+    // Integral arithmetic
+    const __negsi2 = @import("compiler_rt/negXi2.zig").__negsi2;
+    @export(__negsi2, .{ .name = "__negsi2", .linkage = linkage });
+    const __negdi2 = @import("compiler_rt/negXi2.zig").__negdi2;
+    @export(__negdi2, .{ .name = "__negdi2", .linkage = linkage });
+    const __negti2 = @import("compiler_rt/negXi2.zig").__negti2;
+    @export(__negti2, .{ .name = "__negti2", .linkage = linkage });
+    const __mulsi3 = @import("compiler_rt/int.zig").__mulsi3;
+    @export(__mulsi3, .{ .name = "__mulsi3", .linkage = linkage });
+    const __muldi3 = @import("compiler_rt/muldi3.zig").__muldi3;
+    @export(__muldi3, .{ .name = "__muldi3", .linkage = linkage });
+    const __divmoddi4 = @import("compiler_rt/int.zig").__divmoddi4;
+    @export(__divmoddi4, .{ .name = "__divmoddi4", .linkage = linkage });
+    const __divsi3 = @import("compiler_rt/int.zig").__divsi3;
+    @export(__divsi3, .{ .name = "__divsi3", .linkage = linkage });
+    const __divdi3 = @import("compiler_rt/int.zig").__divdi3;
+    @export(__divdi3, .{ .name = "__divdi3", .linkage = linkage });
+    const __udivsi3 = @import("compiler_rt/int.zig").__udivsi3;
+    @export(__udivsi3, .{ .name = "__udivsi3", .linkage = linkage });
+    const __udivdi3 = @import("compiler_rt/int.zig").__udivdi3;
+    @export(__udivdi3, .{ .name = "__udivdi3", .linkage = linkage });
+    const __modsi3 = @import("compiler_rt/int.zig").__modsi3;
+    @export(__modsi3, .{ .name = "__modsi3", .linkage = linkage });
+    const __moddi3 = @import("compiler_rt/int.zig").__moddi3;
+    @export(__moddi3, .{ .name = "__moddi3", .linkage = linkage });
+    const __umodsi3 = @import("compiler_rt/int.zig").__umodsi3;
+    @export(__umodsi3, .{ .name = "__umodsi3", .linkage = linkage });
+    const __umoddi3 = @import("compiler_rt/int.zig").__umoddi3;
+    @export(__umoddi3, .{ .name = "__umoddi3", .linkage = linkage });
+    const __divmodsi4 = @import("compiler_rt/int.zig").__divmodsi4;
+    @export(__divmodsi4, .{ .name = "__divmodsi4", .linkage = linkage });
+    const __udivmodsi4 = @import("compiler_rt/int.zig").__udivmodsi4;
+    @export(__udivmodsi4, .{ .name = "__udivmodsi4", .linkage = linkage });
+
+    // Integral arithmetic with trapping overflow
+    const __absvsi2 = @import("compiler_rt/absv.zig").__absvsi2;
+    @export(__absvsi2, .{ .name = "__absvsi2", .linkage = linkage });
+    const __absvdi2 = @import("compiler_rt/absv.zig").__absvdi2;
+    @export(__absvdi2, .{ .name = "__absvdi2", .linkage = linkage });
+    const __absvti2 = @import("compiler_rt/absv.zig").__absvti2;
+    @export(__absvti2, .{ .name = "__absvti2", .linkage = linkage });
+    const __negvsi2 = @import("compiler_rt/negv.zig").__negvsi2;
+    @export(__negvsi2, .{ .name = "__negvsi2", .linkage = linkage });
+    const __negvdi2 = @import("compiler_rt/negv.zig").__negvdi2;
+    @export(__negvdi2, .{ .name = "__negvdi2", .linkage = linkage });
+    const __negvti2 = @import("compiler_rt/negv.zig").__negvti2;
+    @export(__negvti2, .{ .name = "__negvti2", .linkage = linkage });
+
+    // missing: Integral arithmetic which returns if overflow
+
+    // Integral comparison
+    // (a <  b) => 0
+    // (a == b) => 1
+    // (a >  b) => 2
+    const __cmpsi2 = @import("compiler_rt/cmp.zig").__cmpsi2;
+    @export(__cmpsi2, .{ .name = "__cmpsi2", .linkage = linkage });
+    const __cmpdi2 = @import("compiler_rt/cmp.zig").__cmpdi2;
+    @export(__cmpdi2, .{ .name = "__cmpdi2", .linkage = linkage });
+    const __cmpti2 = @import("compiler_rt/cmp.zig").__cmpti2;
+    @export(__cmpti2, .{ .name = "__cmpti2", .linkage = linkage });
+    const __ucmpsi2 = @import("compiler_rt/cmp.zig").__ucmpsi2;
+    @export(__ucmpsi2, .{ .name = "__ucmpsi2", .linkage = linkage });
+    const __ucmpdi2 = @import("compiler_rt/cmp.zig").__ucmpdi2;
+    @export(__ucmpdi2, .{ .name = "__ucmpdi2", .linkage = linkage });
+    const __ucmpti2 = @import("compiler_rt/cmp.zig").__ucmpti2;
+    @export(__ucmpti2, .{ .name = "__ucmpti2", .linkage = linkage });
+
+    // missing: Floating point raised to integer power
+
+    // missing: Complex arithmetic
+    // (a + ib) * (c + id)
+    // (a + ib) / (c + id)
+
+    const __negsf2 = @import("compiler_rt/negXf2.zig").__negsf2;
+    @export(__negsf2, .{ .name = "__negsf2", .linkage = linkage });
+    const __negdf2 = @import("compiler_rt/negXf2.zig").__negdf2;
+    @export(__negdf2, .{ .name = "__negdf2", .linkage = linkage });
+
+    if (builtin.link_libc and os_tag == .openbsd) {
+        const __emutls_get_address = @import("compiler_rt/emutls.zig").__emutls_get_address;
+        @export(__emutls_get_address, .{ .name = "__emutls_get_address", .linkage = linkage });
+    }
 
-        if ((arch.isARM() or arch.isThumb()) and !is_test) {
-            const __aeabi_unwind_cpp_pr0 = @import("compiler_rt/arm.zig").__aeabi_unwind_cpp_pr0;
-            @export(__aeabi_unwind_cpp_pr0, .{ .name = "__aeabi_unwind_cpp_pr0", .linkage = linkage });
-            const __aeabi_unwind_cpp_pr1 = @import("compiler_rt/arm.zig").__aeabi_unwind_cpp_pr1;
-            @export(__aeabi_unwind_cpp_pr1, .{ .name = "__aeabi_unwind_cpp_pr1", .linkage = linkage });
-            const __aeabi_unwind_cpp_pr2 = @import("compiler_rt/arm.zig").__aeabi_unwind_cpp_pr2;
-            @export(__aeabi_unwind_cpp_pr2, .{ .name = "__aeabi_unwind_cpp_pr2", .linkage = linkage });
-
-            @export(__muldi3, .{ .name = "__aeabi_lmul", .linkage = linkage });
-
-            const __aeabi_ldivmod = @import("compiler_rt/arm.zig").__aeabi_ldivmod;
-            @export(__aeabi_ldivmod, .{ .name = "__aeabi_ldivmod", .linkage = linkage });
-            const __aeabi_uldivmod = @import("compiler_rt/arm.zig").__aeabi_uldivmod;
-            @export(__aeabi_uldivmod, .{ .name = "__aeabi_uldivmod", .linkage = linkage });
-
-            @export(__divsi3, .{ .name = "__aeabi_idiv", .linkage = linkage });
-            const __aeabi_idivmod = @import("compiler_rt/arm.zig").__aeabi_idivmod;
-            @export(__aeabi_idivmod, .{ .name = "__aeabi_idivmod", .linkage = linkage });
-            @export(__udivsi3, .{ .name = "__aeabi_uidiv", .linkage = linkage });
-            const __aeabi_uidivmod = @import("compiler_rt/arm.zig").__aeabi_uidivmod;
-            @export(__aeabi_uidivmod, .{ .name = "__aeabi_uidivmod", .linkage = linkage });
-
-            const __aeabi_memcpy = @import("compiler_rt/arm.zig").__aeabi_memcpy;
-            @export(__aeabi_memcpy, .{ .name = "__aeabi_memcpy", .linkage = linkage });
-            @export(__aeabi_memcpy, .{ .name = "__aeabi_memcpy4", .linkage = linkage });
-            @export(__aeabi_memcpy, .{ .name = "__aeabi_memcpy8", .linkage = linkage });
-
-            const __aeabi_memmove = @import("compiler_rt/arm.zig").__aeabi_memmove;
-            @export(__aeabi_memmove, .{ .name = "__aeabi_memmove", .linkage = linkage });
-            @export(__aeabi_memmove, .{ .name = "__aeabi_memmove4", .linkage = linkage });
-            @export(__aeabi_memmove, .{ .name = "__aeabi_memmove8", .linkage = linkage });
-
-            const __aeabi_memset = @import("compiler_rt/arm.zig").__aeabi_memset;
-            @export(__aeabi_memset, .{ .name = "__aeabi_memset", .linkage = linkage });
-            @export(__aeabi_memset, .{ .name = "__aeabi_memset4", .linkage = linkage });
-            @export(__aeabi_memset, .{ .name = "__aeabi_memset8", .linkage = linkage });
-
-            const __aeabi_memclr = @import("compiler_rt/arm.zig").__aeabi_memclr;
-            @export(__aeabi_memclr, .{ .name = "__aeabi_memclr", .linkage = linkage });
-            @export(__aeabi_memclr, .{ .name = "__aeabi_memclr4", .linkage = linkage });
-            @export(__aeabi_memclr, .{ .name = "__aeabi_memclr8", .linkage = linkage });
-
-            if (os_tag == .linux) {
-                const __aeabi_read_tp = @import("compiler_rt/arm.zig").__aeabi_read_tp;
-                @export(__aeabi_read_tp, .{ .name = "__aeabi_read_tp", .linkage = linkage });
-            }
-
-            const __aeabi_f2d = @import("compiler_rt/extendXfYf2.zig").__aeabi_f2d;
-            @export(__aeabi_f2d, .{ .name = "__aeabi_f2d", .linkage = linkage });
-            const __aeabi_i2d = @import("compiler_rt/floatsiXf.zig").__aeabi_i2d;
-            @export(__aeabi_i2d, .{ .name = "__aeabi_i2d", .linkage = linkage });
-            const __aeabi_l2d = @import("compiler_rt/floatdidf.zig").__aeabi_l2d;
-            @export(__aeabi_l2d, .{ .name = "__aeabi_l2d", .linkage = linkage });
-            const __aeabi_l2f = @import("compiler_rt/floatXisf.zig").__aeabi_l2f;
-            @export(__aeabi_l2f, .{ .name = "__aeabi_l2f", .linkage = linkage });
-            const __aeabi_ui2d = @import("compiler_rt/floatunsidf.zig").__aeabi_ui2d;
-            @export(__aeabi_ui2d, .{ .name = "__aeabi_ui2d", .linkage = linkage });
-            const __aeabi_ul2d = @import("compiler_rt/floatundidf.zig").__aeabi_ul2d;
-            @export(__aeabi_ul2d, .{ .name = "__aeabi_ul2d", .linkage = linkage });
-            const __aeabi_ui2f = @import("compiler_rt/floatunsisf.zig").__aeabi_ui2f;
-            @export(__aeabi_ui2f, .{ .name = "__aeabi_ui2f", .linkage = linkage });
-            const __aeabi_ul2f = @import("compiler_rt/floatundisf.zig").__aeabi_ul2f;
-            @export(__aeabi_ul2f, .{ .name = "__aeabi_ul2f", .linkage = linkage });
-
-            const __aeabi_fneg = @import("compiler_rt/negXf2.zig").__aeabi_fneg;
-            @export(__aeabi_fneg, .{ .name = "__aeabi_fneg", .linkage = linkage });
-            const __aeabi_dneg = @import("compiler_rt/negXf2.zig").__aeabi_dneg;
-            @export(__aeabi_dneg, .{ .name = "__aeabi_dneg", .linkage = linkage });
-
-            const __aeabi_fmul = @import("compiler_rt/mulXf3.zig").__aeabi_fmul;
-            @export(__aeabi_fmul, .{ .name = "__aeabi_fmul", .linkage = linkage });
-            const __aeabi_dmul = @import("compiler_rt/mulXf3.zig").__aeabi_dmul;
-            @export(__aeabi_dmul, .{ .name = "__aeabi_dmul", .linkage = linkage });
-
-            const __aeabi_d2h = @import("compiler_rt/truncXfYf2.zig").__aeabi_d2h;
-            @export(__aeabi_d2h, .{ .name = "__aeabi_d2h", .linkage = linkage });
-
-            const __aeabi_f2ulz = @import("compiler_rt/fixunssfdi.zig").__aeabi_f2ulz;
-            @export(__aeabi_f2ulz, .{ .name = "__aeabi_f2ulz", .linkage = linkage });
-            const __aeabi_d2ulz = @import("compiler_rt/fixunsdfdi.zig").__aeabi_d2ulz;
-            @export(__aeabi_d2ulz, .{ .name = "__aeabi_d2ulz", .linkage = linkage });
-
-            const __aeabi_f2lz = @import("compiler_rt/fixsfdi.zig").__aeabi_f2lz;
-            @export(__aeabi_f2lz, .{ .name = "__aeabi_f2lz", .linkage = linkage });
-            const __aeabi_d2lz = @import("compiler_rt/fixdfdi.zig").__aeabi_d2lz;
-            @export(__aeabi_d2lz, .{ .name = "__aeabi_d2lz", .linkage = linkage });
-
-            const __aeabi_d2uiz = @import("compiler_rt/fixunsdfsi.zig").__aeabi_d2uiz;
-            @export(__aeabi_d2uiz, .{ .name = "__aeabi_d2uiz", .linkage = linkage });
-
-            const __aeabi_h2f = @import("compiler_rt/extendXfYf2.zig").__aeabi_h2f;
-            @export(__aeabi_h2f, .{ .name = "__aeabi_h2f", .linkage = linkage });
-            const __aeabi_f2h = @import("compiler_rt/truncXfYf2.zig").__aeabi_f2h;
-            @export(__aeabi_f2h, .{ .name = "__aeabi_f2h", .linkage = linkage });
-
-            const __aeabi_i2f = @import("compiler_rt/floatsiXf.zig").__aeabi_i2f;
-            @export(__aeabi_i2f, .{ .name = "__aeabi_i2f", .linkage = linkage });
-            const __aeabi_d2f = @import("compiler_rt/truncXfYf2.zig").__aeabi_d2f;
-            @export(__aeabi_d2f, .{ .name = "__aeabi_d2f", .linkage = linkage });
-
-            const __aeabi_fadd = @import("compiler_rt/addXf3.zig").__aeabi_fadd;
-            @export(__aeabi_fadd, .{ .name = "__aeabi_fadd", .linkage = linkage });
-            const __aeabi_dadd = @import("compiler_rt/addXf3.zig").__aeabi_dadd;
-            @export(__aeabi_dadd, .{ .name = "__aeabi_dadd", .linkage = linkage });
-            const __aeabi_fsub = @import("compiler_rt/addXf3.zig").__aeabi_fsub;
-            @export(__aeabi_fsub, .{ .name = "__aeabi_fsub", .linkage = linkage });
-            const __aeabi_dsub = @import("compiler_rt/addXf3.zig").__aeabi_dsub;
-            @export(__aeabi_dsub, .{ .name = "__aeabi_dsub", .linkage = linkage });
-
-            const __aeabi_f2uiz = @import("compiler_rt/fixunssfsi.zig").__aeabi_f2uiz;
-            @export(__aeabi_f2uiz, .{ .name = "__aeabi_f2uiz", .linkage = linkage });
-
-            const __aeabi_f2iz = @import("compiler_rt/fixsfsi.zig").__aeabi_f2iz;
-            @export(__aeabi_f2iz, .{ .name = "__aeabi_f2iz", .linkage = linkage });
-            const __aeabi_d2iz = @import("compiler_rt/fixdfsi.zig").__aeabi_d2iz;
-            @export(__aeabi_d2iz, .{ .name = "__aeabi_d2iz", .linkage = linkage });
-
-            const __aeabi_fdiv = @import("compiler_rt/divsf3.zig").__aeabi_fdiv;
-            @export(__aeabi_fdiv, .{ .name = "__aeabi_fdiv", .linkage = linkage });
-            const __aeabi_ddiv = @import("compiler_rt/divdf3.zig").__aeabi_ddiv;
-            @export(__aeabi_ddiv, .{ .name = "__aeabi_ddiv", .linkage = linkage });
-
-            const __aeabi_llsl = @import("compiler_rt/shift.zig").__aeabi_llsl;
-            @export(__aeabi_llsl, .{ .name = "__aeabi_llsl", .linkage = linkage });
-            const __aeabi_lasr = @import("compiler_rt/shift.zig").__aeabi_lasr;
-            @export(__aeabi_lasr, .{ .name = "__aeabi_lasr", .linkage = linkage });
-            const __aeabi_llsr = @import("compiler_rt/shift.zig").__aeabi_llsr;
-            @export(__aeabi_llsr, .{ .name = "__aeabi_llsr", .linkage = linkage });
-
-            const __aeabi_fcmpeq = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpeq;
-            @export(__aeabi_fcmpeq, .{ .name = "__aeabi_fcmpeq", .linkage = linkage });
-            const __aeabi_fcmplt = @import("compiler_rt/compareXf2.zig").__aeabi_fcmplt;
-            @export(__aeabi_fcmplt, .{ .name = "__aeabi_fcmplt", .linkage = linkage });
-            const __aeabi_fcmple = @import("compiler_rt/compareXf2.zig").__aeabi_fcmple;
-            @export(__aeabi_fcmple, .{ .name = "__aeabi_fcmple", .linkage = linkage });
-            const __aeabi_fcmpge = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpge;
-            @export(__aeabi_fcmpge, .{ .name = "__aeabi_fcmpge", .linkage = linkage });
-            const __aeabi_fcmpgt = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpgt;
-            @export(__aeabi_fcmpgt, .{ .name = "__aeabi_fcmpgt", .linkage = linkage });
-            const __aeabi_fcmpun = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpun;
-            @export(__aeabi_fcmpun, .{ .name = "__aeabi_fcmpun", .linkage = linkage });
-
-            const __aeabi_dcmpeq = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpeq;
-            @export(__aeabi_dcmpeq, .{ .name = "__aeabi_dcmpeq", .linkage = linkage });
-            const __aeabi_dcmplt = @import("compiler_rt/compareXf2.zig").__aeabi_dcmplt;
-            @export(__aeabi_dcmplt, .{ .name = "__aeabi_dcmplt", .linkage = linkage });
-            const __aeabi_dcmple = @import("compiler_rt/compareXf2.zig").__aeabi_dcmple;
-            @export(__aeabi_dcmple, .{ .name = "__aeabi_dcmple", .linkage = linkage });
-            const __aeabi_dcmpge = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpge;
-            @export(__aeabi_dcmpge, .{ .name = "__aeabi_dcmpge", .linkage = linkage });
-            const __aeabi_dcmpgt = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpgt;
-            @export(__aeabi_dcmpgt, .{ .name = "__aeabi_dcmpgt", .linkage = linkage });
-            const __aeabi_dcmpun = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpun;
-            @export(__aeabi_dcmpun, .{ .name = "__aeabi_dcmpun", .linkage = linkage });
+    if ((arch.isARM() or arch.isThumb()) and !is_test) {
+        const __aeabi_unwind_cpp_pr0 = @import("compiler_rt/arm.zig").__aeabi_unwind_cpp_pr0;
+        @export(__aeabi_unwind_cpp_pr0, .{ .name = "__aeabi_unwind_cpp_pr0", .linkage = linkage });
+        const __aeabi_unwind_cpp_pr1 = @import("compiler_rt/arm.zig").__aeabi_unwind_cpp_pr1;
+        @export(__aeabi_unwind_cpp_pr1, .{ .name = "__aeabi_unwind_cpp_pr1", .linkage = linkage });
+        const __aeabi_unwind_cpp_pr2 = @import("compiler_rt/arm.zig").__aeabi_unwind_cpp_pr2;
+        @export(__aeabi_unwind_cpp_pr2, .{ .name = "__aeabi_unwind_cpp_pr2", .linkage = linkage });
+
+        @export(__muldi3, .{ .name = "__aeabi_lmul", .linkage = linkage });
+
+        const __aeabi_ldivmod = @import("compiler_rt/arm.zig").__aeabi_ldivmod;
+        @export(__aeabi_ldivmod, .{ .name = "__aeabi_ldivmod", .linkage = linkage });
+        const __aeabi_uldivmod = @import("compiler_rt/arm.zig").__aeabi_uldivmod;
+        @export(__aeabi_uldivmod, .{ .name = "__aeabi_uldivmod", .linkage = linkage });
+
+        @export(__divsi3, .{ .name = "__aeabi_idiv", .linkage = linkage });
+        const __aeabi_idivmod = @import("compiler_rt/arm.zig").__aeabi_idivmod;
+        @export(__aeabi_idivmod, .{ .name = "__aeabi_idivmod", .linkage = linkage });
+        @export(__udivsi3, .{ .name = "__aeabi_uidiv", .linkage = linkage });
+        const __aeabi_uidivmod = @import("compiler_rt/arm.zig").__aeabi_uidivmod;
+        @export(__aeabi_uidivmod, .{ .name = "__aeabi_uidivmod", .linkage = linkage });
+
+        const __aeabi_memcpy = @import("compiler_rt/arm.zig").__aeabi_memcpy;
+        @export(__aeabi_memcpy, .{ .name = "__aeabi_memcpy", .linkage = linkage });
+        @export(__aeabi_memcpy, .{ .name = "__aeabi_memcpy4", .linkage = linkage });
+        @export(__aeabi_memcpy, .{ .name = "__aeabi_memcpy8", .linkage = linkage });
+
+        const __aeabi_memmove = @import("compiler_rt/arm.zig").__aeabi_memmove;
+        @export(__aeabi_memmove, .{ .name = "__aeabi_memmove", .linkage = linkage });
+        @export(__aeabi_memmove, .{ .name = "__aeabi_memmove4", .linkage = linkage });
+        @export(__aeabi_memmove, .{ .name = "__aeabi_memmove8", .linkage = linkage });
+
+        const __aeabi_memset = @import("compiler_rt/arm.zig").__aeabi_memset;
+        @export(__aeabi_memset, .{ .name = "__aeabi_memset", .linkage = linkage });
+        @export(__aeabi_memset, .{ .name = "__aeabi_memset4", .linkage = linkage });
+        @export(__aeabi_memset, .{ .name = "__aeabi_memset8", .linkage = linkage });
+
+        const __aeabi_memclr = @import("compiler_rt/arm.zig").__aeabi_memclr;
+        @export(__aeabi_memclr, .{ .name = "__aeabi_memclr", .linkage = linkage });
+        @export(__aeabi_memclr, .{ .name = "__aeabi_memclr4", .linkage = linkage });
+        @export(__aeabi_memclr, .{ .name = "__aeabi_memclr8", .linkage = linkage });
+
+        if (os_tag == .linux) {
+            const __aeabi_read_tp = @import("compiler_rt/arm.zig").__aeabi_read_tp;
+            @export(__aeabi_read_tp, .{ .name = "__aeabi_read_tp", .linkage = linkage });
         }
 
-        if (arch == .i386 and abi == .msvc) {
-            // Don't let LLVM apply the stdcall name mangling on those MSVC builtins
-            const _alldiv = @import("compiler_rt/aulldiv.zig")._alldiv;
-            @export(_alldiv, .{ .name = "\x01__alldiv", .linkage = strong_linkage });
-            const _aulldiv = @import("compiler_rt/aulldiv.zig")._aulldiv;
-            @export(_aulldiv, .{ .name = "\x01__aulldiv", .linkage = strong_linkage });
-            const _allrem = @import("compiler_rt/aullrem.zig")._allrem;
-            @export(_allrem, .{ .name = "\x01__allrem", .linkage = strong_linkage });
-            const _aullrem = @import("compiler_rt/aullrem.zig")._aullrem;
-            @export(_aullrem, .{ .name = "\x01__aullrem", .linkage = strong_linkage });
-        }
+        const __aeabi_f2d = @import("compiler_rt/extendXfYf2.zig").__aeabi_f2d;
+        @export(__aeabi_f2d, .{ .name = "__aeabi_f2d", .linkage = linkage });
+        const __aeabi_i2d = @import("compiler_rt/floatsiXf.zig").__aeabi_i2d;
+        @export(__aeabi_i2d, .{ .name = "__aeabi_i2d", .linkage = linkage });
+        const __aeabi_l2d = @import("compiler_rt/floatdidf.zig").__aeabi_l2d;
+        @export(__aeabi_l2d, .{ .name = "__aeabi_l2d", .linkage = linkage });
+        const __aeabi_l2f = @import("compiler_rt/floatXisf.zig").__aeabi_l2f;
+        @export(__aeabi_l2f, .{ .name = "__aeabi_l2f", .linkage = linkage });
+        const __aeabi_ui2d = @import("compiler_rt/floatunsidf.zig").__aeabi_ui2d;
+        @export(__aeabi_ui2d, .{ .name = "__aeabi_ui2d", .linkage = linkage });
+        const __aeabi_ul2d = @import("compiler_rt/floatundidf.zig").__aeabi_ul2d;
+        @export(__aeabi_ul2d, .{ .name = "__aeabi_ul2d", .linkage = linkage });
+        const __aeabi_ui2f = @import("compiler_rt/floatunsisf.zig").__aeabi_ui2f;
+        @export(__aeabi_ui2f, .{ .name = "__aeabi_ui2f", .linkage = linkage });
+        const __aeabi_ul2f = @import("compiler_rt/floatundisf.zig").__aeabi_ul2f;
+        @export(__aeabi_ul2f, .{ .name = "__aeabi_ul2f", .linkage = linkage });
+
+        const __aeabi_fneg = @import("compiler_rt/negXf2.zig").__aeabi_fneg;
+        @export(__aeabi_fneg, .{ .name = "__aeabi_fneg", .linkage = linkage });
+        const __aeabi_dneg = @import("compiler_rt/negXf2.zig").__aeabi_dneg;
+        @export(__aeabi_dneg, .{ .name = "__aeabi_dneg", .linkage = linkage });
+
+        const __aeabi_fmul = @import("compiler_rt/mulXf3.zig").__aeabi_fmul;
+        @export(__aeabi_fmul, .{ .name = "__aeabi_fmul", .linkage = linkage });
+        const __aeabi_dmul = @import("compiler_rt/mulXf3.zig").__aeabi_dmul;
+        @export(__aeabi_dmul, .{ .name = "__aeabi_dmul", .linkage = linkage });
+
+        const __aeabi_d2h = @import("compiler_rt/truncXfYf2.zig").__aeabi_d2h;
+        @export(__aeabi_d2h, .{ .name = "__aeabi_d2h", .linkage = linkage });
+
+        const __aeabi_f2ulz = @import("compiler_rt/fixunssfdi.zig").__aeabi_f2ulz;
+        @export(__aeabi_f2ulz, .{ .name = "__aeabi_f2ulz", .linkage = linkage });
+        const __aeabi_d2ulz = @import("compiler_rt/fixunsdfdi.zig").__aeabi_d2ulz;
+        @export(__aeabi_d2ulz, .{ .name = "__aeabi_d2ulz", .linkage = linkage });
+
+        const __aeabi_f2lz = @import("compiler_rt/fixsfdi.zig").__aeabi_f2lz;
+        @export(__aeabi_f2lz, .{ .name = "__aeabi_f2lz", .linkage = linkage });
+        const __aeabi_d2lz = @import("compiler_rt/fixdfdi.zig").__aeabi_d2lz;
+        @export(__aeabi_d2lz, .{ .name = "__aeabi_d2lz", .linkage = linkage });
+
+        const __aeabi_d2uiz = @import("compiler_rt/fixunsdfsi.zig").__aeabi_d2uiz;
+        @export(__aeabi_d2uiz, .{ .name = "__aeabi_d2uiz", .linkage = linkage });
+
+        const __aeabi_h2f = @import("compiler_rt/extendXfYf2.zig").__aeabi_h2f;
+        @export(__aeabi_h2f, .{ .name = "__aeabi_h2f", .linkage = linkage });
+        const __aeabi_f2h = @import("compiler_rt/truncXfYf2.zig").__aeabi_f2h;
+        @export(__aeabi_f2h, .{ .name = "__aeabi_f2h", .linkage = linkage });
+
+        const __aeabi_i2f = @import("compiler_rt/floatsiXf.zig").__aeabi_i2f;
+        @export(__aeabi_i2f, .{ .name = "__aeabi_i2f", .linkage = linkage });
+        const __aeabi_d2f = @import("compiler_rt/truncXfYf2.zig").__aeabi_d2f;
+        @export(__aeabi_d2f, .{ .name = "__aeabi_d2f", .linkage = linkage });
+
+        const __aeabi_fadd = @import("compiler_rt/addXf3.zig").__aeabi_fadd;
+        @export(__aeabi_fadd, .{ .name = "__aeabi_fadd", .linkage = linkage });
+        const __aeabi_dadd = @import("compiler_rt/addXf3.zig").__aeabi_dadd;
+        @export(__aeabi_dadd, .{ .name = "__aeabi_dadd", .linkage = linkage });
+        const __aeabi_fsub = @import("compiler_rt/addXf3.zig").__aeabi_fsub;
+        @export(__aeabi_fsub, .{ .name = "__aeabi_fsub", .linkage = linkage });
+        const __aeabi_dsub = @import("compiler_rt/addXf3.zig").__aeabi_dsub;
+        @export(__aeabi_dsub, .{ .name = "__aeabi_dsub", .linkage = linkage });
+
+        const __aeabi_f2uiz = @import("compiler_rt/fixunssfsi.zig").__aeabi_f2uiz;
+        @export(__aeabi_f2uiz, .{ .name = "__aeabi_f2uiz", .linkage = linkage });
+
+        const __aeabi_f2iz = @import("compiler_rt/fixsfsi.zig").__aeabi_f2iz;
+        @export(__aeabi_f2iz, .{ .name = "__aeabi_f2iz", .linkage = linkage });
+        const __aeabi_d2iz = @import("compiler_rt/fixdfsi.zig").__aeabi_d2iz;
+        @export(__aeabi_d2iz, .{ .name = "__aeabi_d2iz", .linkage = linkage });
+
+        const __aeabi_fdiv = @import("compiler_rt/divsf3.zig").__aeabi_fdiv;
+        @export(__aeabi_fdiv, .{ .name = "__aeabi_fdiv", .linkage = linkage });
+        const __aeabi_ddiv = @import("compiler_rt/divdf3.zig").__aeabi_ddiv;
+        @export(__aeabi_ddiv, .{ .name = "__aeabi_ddiv", .linkage = linkage });
+
+        const __aeabi_llsl = @import("compiler_rt/shift.zig").__aeabi_llsl;
+        @export(__aeabi_llsl, .{ .name = "__aeabi_llsl", .linkage = linkage });
+        const __aeabi_lasr = @import("compiler_rt/shift.zig").__aeabi_lasr;
+        @export(__aeabi_lasr, .{ .name = "__aeabi_lasr", .linkage = linkage });
+        const __aeabi_llsr = @import("compiler_rt/shift.zig").__aeabi_llsr;
+        @export(__aeabi_llsr, .{ .name = "__aeabi_llsr", .linkage = linkage });
+
+        const __aeabi_fcmpeq = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpeq;
+        @export(__aeabi_fcmpeq, .{ .name = "__aeabi_fcmpeq", .linkage = linkage });
+        const __aeabi_fcmplt = @import("compiler_rt/compareXf2.zig").__aeabi_fcmplt;
+        @export(__aeabi_fcmplt, .{ .name = "__aeabi_fcmplt", .linkage = linkage });
+        const __aeabi_fcmple = @import("compiler_rt/compareXf2.zig").__aeabi_fcmple;
+        @export(__aeabi_fcmple, .{ .name = "__aeabi_fcmple", .linkage = linkage });
+        const __aeabi_fcmpge = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpge;
+        @export(__aeabi_fcmpge, .{ .name = "__aeabi_fcmpge", .linkage = linkage });
+        const __aeabi_fcmpgt = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpgt;
+        @export(__aeabi_fcmpgt, .{ .name = "__aeabi_fcmpgt", .linkage = linkage });
+        const __aeabi_fcmpun = @import("compiler_rt/compareXf2.zig").__aeabi_fcmpun;
+        @export(__aeabi_fcmpun, .{ .name = "__aeabi_fcmpun", .linkage = linkage });
+
+        const __aeabi_dcmpeq = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpeq;
+        @export(__aeabi_dcmpeq, .{ .name = "__aeabi_dcmpeq", .linkage = linkage });
+        const __aeabi_dcmplt = @import("compiler_rt/compareXf2.zig").__aeabi_dcmplt;
+        @export(__aeabi_dcmplt, .{ .name = "__aeabi_dcmplt", .linkage = linkage });
+        const __aeabi_dcmple = @import("compiler_rt/compareXf2.zig").__aeabi_dcmple;
+        @export(__aeabi_dcmple, .{ .name = "__aeabi_dcmple", .linkage = linkage });
+        const __aeabi_dcmpge = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpge;
+        @export(__aeabi_dcmpge, .{ .name = "__aeabi_dcmpge", .linkage = linkage });
+        const __aeabi_dcmpgt = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpgt;
+        @export(__aeabi_dcmpgt, .{ .name = "__aeabi_dcmpgt", .linkage = linkage });
+        const __aeabi_dcmpun = @import("compiler_rt/compareXf2.zig").__aeabi_dcmpun;
+        @export(__aeabi_dcmpun, .{ .name = "__aeabi_dcmpun", .linkage = linkage });
+    }
 
-        if (arch.isSPARC()) {
-            // SPARC systems use a different naming scheme
-            const _Qp_add = @import("compiler_rt/sparc.zig")._Qp_add;
-            @export(_Qp_add, .{ .name = "_Qp_add", .linkage = linkage });
-            const _Qp_div = @import("compiler_rt/sparc.zig")._Qp_div;
-            @export(_Qp_div, .{ .name = "_Qp_div", .linkage = linkage });
-            const _Qp_mul = @import("compiler_rt/sparc.zig")._Qp_mul;
-            @export(_Qp_mul, .{ .name = "_Qp_mul", .linkage = linkage });
-            const _Qp_sub = @import("compiler_rt/sparc.zig")._Qp_sub;
-            @export(_Qp_sub, .{ .name = "_Qp_sub", .linkage = linkage });
-
-            const _Qp_cmp = @import("compiler_rt/sparc.zig")._Qp_cmp;
-            @export(_Qp_cmp, .{ .name = "_Qp_cmp", .linkage = linkage });
-            const _Qp_feq = @import("compiler_rt/sparc.zig")._Qp_feq;
-            @export(_Qp_feq, .{ .name = "_Qp_feq", .linkage = linkage });
-            const _Qp_fne = @import("compiler_rt/sparc.zig")._Qp_fne;
-            @export(_Qp_fne, .{ .name = "_Qp_fne", .linkage = linkage });
-            const _Qp_flt = @import("compiler_rt/sparc.zig")._Qp_flt;
-            @export(_Qp_flt, .{ .name = "_Qp_flt", .linkage = linkage });
-            const _Qp_fle = @import("compiler_rt/sparc.zig")._Qp_fle;
-            @export(_Qp_fle, .{ .name = "_Qp_fle", .linkage = linkage });
-            const _Qp_fgt = @import("compiler_rt/sparc.zig")._Qp_fgt;
-            @export(_Qp_fgt, .{ .name = "_Qp_fgt", .linkage = linkage });
-            const _Qp_fge = @import("compiler_rt/sparc.zig")._Qp_fge;
-            @export(_Qp_fge, .{ .name = "_Qp_fge", .linkage = linkage });
-
-            const _Qp_itoq = @import("compiler_rt/sparc.zig")._Qp_itoq;
-            @export(_Qp_itoq, .{ .name = "_Qp_itoq", .linkage = linkage });
-            const _Qp_uitoq = @import("compiler_rt/sparc.zig")._Qp_uitoq;
-            @export(_Qp_uitoq, .{ .name = "_Qp_uitoq", .linkage = linkage });
-            const _Qp_xtoq = @import("compiler_rt/sparc.zig")._Qp_xtoq;
-            @export(_Qp_xtoq, .{ .name = "_Qp_xtoq", .linkage = linkage });
-            const _Qp_uxtoq = @import("compiler_rt/sparc.zig")._Qp_uxtoq;
-            @export(_Qp_uxtoq, .{ .name = "_Qp_uxtoq", .linkage = linkage });
-            const _Qp_stoq = @import("compiler_rt/sparc.zig")._Qp_stoq;
-            @export(_Qp_stoq, .{ .name = "_Qp_stoq", .linkage = linkage });
-            const _Qp_dtoq = @import("compiler_rt/sparc.zig")._Qp_dtoq;
-            @export(_Qp_dtoq, .{ .name = "_Qp_dtoq", .linkage = linkage });
-            const _Qp_qtoi = @import("compiler_rt/sparc.zig")._Qp_qtoi;
-            @export(_Qp_qtoi, .{ .name = "_Qp_qtoi", .linkage = linkage });
-            const _Qp_qtoui = @import("compiler_rt/sparc.zig")._Qp_qtoui;
-            @export(_Qp_qtoui, .{ .name = "_Qp_qtoui", .linkage = linkage });
-            const _Qp_qtox = @import("compiler_rt/sparc.zig")._Qp_qtox;
-            @export(_Qp_qtox, .{ .name = "_Qp_qtox", .linkage = linkage });
-            const _Qp_qtoux = @import("compiler_rt/sparc.zig")._Qp_qtoux;
-            @export(_Qp_qtoux, .{ .name = "_Qp_qtoux", .linkage = linkage });
-            const _Qp_qtos = @import("compiler_rt/sparc.zig")._Qp_qtos;
-            @export(_Qp_qtos, .{ .name = "_Qp_qtos", .linkage = linkage });
-            const _Qp_qtod = @import("compiler_rt/sparc.zig")._Qp_qtod;
-            @export(_Qp_qtod, .{ .name = "_Qp_qtod", .linkage = linkage });
-        }
+    if (arch == .i386 and abi == .msvc) {
+        // Don't let LLVM apply the stdcall name mangling on those MSVC builtins
+        const _alldiv = @import("compiler_rt/aulldiv.zig")._alldiv;
+        @export(_alldiv, .{ .name = "\x01__alldiv", .linkage = strong_linkage });
+        const _aulldiv = @import("compiler_rt/aulldiv.zig")._aulldiv;
+        @export(_aulldiv, .{ .name = "\x01__aulldiv", .linkage = strong_linkage });
+        const _allrem = @import("compiler_rt/aullrem.zig")._allrem;
+        @export(_allrem, .{ .name = "\x01__allrem", .linkage = strong_linkage });
+        const _aullrem = @import("compiler_rt/aullrem.zig")._aullrem;
+        @export(_aullrem, .{ .name = "\x01__aullrem", .linkage = strong_linkage });
+    }
 
-        if ((arch == .powerpc or arch.isPPC64()) and !is_test) {
-            @export(__addtf3, .{ .name = "__addkf3", .linkage = linkage });
-            @export(__subtf3, .{ .name = "__subkf3", .linkage = linkage });
-            @export(__multf3, .{ .name = "__mulkf3", .linkage = linkage });
-            @export(__divtf3, .{ .name = "__divkf3", .linkage = linkage });
-            @export(__extendsftf2, .{ .name = "__extendsfkf2", .linkage = linkage });
-            @export(__extenddftf2, .{ .name = "__extenddfkf2", .linkage = linkage });
-            @export(__trunctfsf2, .{ .name = "__trunckfsf2", .linkage = linkage });
-            @export(__trunctfdf2, .{ .name = "__trunckfdf2", .linkage = linkage });
-            @export(__fixtfdi, .{ .name = "__fixkfdi", .linkage = linkage });
-            @export(__fixtfsi, .{ .name = "__fixkfsi", .linkage = linkage });
-            @export(__fixunstfsi, .{ .name = "__fixunskfsi", .linkage = linkage });
-            @export(__fixunstfdi, .{ .name = "__fixunskfdi", .linkage = linkage });
-            @export(__floatsitf, .{ .name = "__floatsikf", .linkage = linkage });
-            @export(__floatditf, .{ .name = "__floatdikf", .linkage = linkage });
-            @export(__floatunditf, .{ .name = "__floatundikf", .linkage = linkage });
-            @export(__floatunsitf, .{ .name = "__floatunsikf", .linkage = linkage });
-
-            @export(__letf2, .{ .name = "__eqkf2", .linkage = linkage });
-            @export(__letf2, .{ .name = "__nekf2", .linkage = linkage });
-            @export(__getf2, .{ .name = "__gekf2", .linkage = linkage });
-            @export(__letf2, .{ .name = "__ltkf2", .linkage = linkage });
-            @export(__letf2, .{ .name = "__lekf2", .linkage = linkage });
-            @export(__getf2, .{ .name = "__gtkf2", .linkage = linkage });
-            @export(__unordtf2, .{ .name = "__unordkf2", .linkage = linkage });
-        }
+    if (arch.isSPARC()) {
+        // SPARC systems use a different naming scheme
+        const _Qp_add = @import("compiler_rt/sparc.zig")._Qp_add;
+        @export(_Qp_add, .{ .name = "_Qp_add", .linkage = linkage });
+        const _Qp_div = @import("compiler_rt/sparc.zig")._Qp_div;
+        @export(_Qp_div, .{ .name = "_Qp_div", .linkage = linkage });
+        const _Qp_mul = @import("compiler_rt/sparc.zig")._Qp_mul;
+        @export(_Qp_mul, .{ .name = "_Qp_mul", .linkage = linkage });
+        const _Qp_sub = @import("compiler_rt/sparc.zig")._Qp_sub;
+        @export(_Qp_sub, .{ .name = "_Qp_sub", .linkage = linkage });
+
+        const _Qp_cmp = @import("compiler_rt/sparc.zig")._Qp_cmp;
+        @export(_Qp_cmp, .{ .name = "_Qp_cmp", .linkage = linkage });
+        const _Qp_feq = @import("compiler_rt/sparc.zig")._Qp_feq;
+        @export(_Qp_feq, .{ .name = "_Qp_feq", .linkage = linkage });
+        const _Qp_fne = @import("compiler_rt/sparc.zig")._Qp_fne;
+        @export(_Qp_fne, .{ .name = "_Qp_fne", .linkage = linkage });
+        const _Qp_flt = @import("compiler_rt/sparc.zig")._Qp_flt;
+        @export(_Qp_flt, .{ .name = "_Qp_flt", .linkage = linkage });
+        const _Qp_fle = @import("compiler_rt/sparc.zig")._Qp_fle;
+        @export(_Qp_fle, .{ .name = "_Qp_fle", .linkage = linkage });
+        const _Qp_fgt = @import("compiler_rt/sparc.zig")._Qp_fgt;
+        @export(_Qp_fgt, .{ .name = "_Qp_fgt", .linkage = linkage });
+        const _Qp_fge = @import("compiler_rt/sparc.zig")._Qp_fge;
+        @export(_Qp_fge, .{ .name = "_Qp_fge", .linkage = linkage });
+
+        const _Qp_itoq = @import("compiler_rt/sparc.zig")._Qp_itoq;
+        @export(_Qp_itoq, .{ .name = "_Qp_itoq", .linkage = linkage });
+        const _Qp_uitoq = @import("compiler_rt/sparc.zig")._Qp_uitoq;
+        @export(_Qp_uitoq, .{ .name = "_Qp_uitoq", .linkage = linkage });
+        const _Qp_xtoq = @import("compiler_rt/sparc.zig")._Qp_xtoq;
+        @export(_Qp_xtoq, .{ .name = "_Qp_xtoq", .linkage = linkage });
+        const _Qp_uxtoq = @import("compiler_rt/sparc.zig")._Qp_uxtoq;
+        @export(_Qp_uxtoq, .{ .name = "_Qp_uxtoq", .linkage = linkage });
+        const _Qp_stoq = @import("compiler_rt/sparc.zig")._Qp_stoq;
+        @export(_Qp_stoq, .{ .name = "_Qp_stoq", .linkage = linkage });
+        const _Qp_dtoq = @import("compiler_rt/sparc.zig")._Qp_dtoq;
+        @export(_Qp_dtoq, .{ .name = "_Qp_dtoq", .linkage = linkage });
+        const _Qp_qtoi = @import("compiler_rt/sparc.zig")._Qp_qtoi;
+        @export(_Qp_qtoi, .{ .name = "_Qp_qtoi", .linkage = linkage });
+        const _Qp_qtoui = @import("compiler_rt/sparc.zig")._Qp_qtoui;
+        @export(_Qp_qtoui, .{ .name = "_Qp_qtoui", .linkage = linkage });
+        const _Qp_qtox = @import("compiler_rt/sparc.zig")._Qp_qtox;
+        @export(_Qp_qtox, .{ .name = "_Qp_qtox", .linkage = linkage });
+        const _Qp_qtoux = @import("compiler_rt/sparc.zig")._Qp_qtoux;
+        @export(_Qp_qtoux, .{ .name = "_Qp_qtoux", .linkage = linkage });
+        const _Qp_qtos = @import("compiler_rt/sparc.zig")._Qp_qtos;
+        @export(_Qp_qtos, .{ .name = "_Qp_qtos", .linkage = linkage });
+        const _Qp_qtod = @import("compiler_rt/sparc.zig")._Qp_qtod;
+        @export(_Qp_qtod, .{ .name = "_Qp_qtod", .linkage = linkage });
+    }
 
-        _ = @import("compiler_rt/atomics.zig");
+    if ((arch == .powerpc or arch.isPPC64()) and !is_test) {
+        @export(__addtf3, .{ .name = "__addkf3", .linkage = linkage });
+        @export(__subtf3, .{ .name = "__subkf3", .linkage = linkage });
+        @export(__multf3, .{ .name = "__mulkf3", .linkage = linkage });
+        @export(__divtf3, .{ .name = "__divkf3", .linkage = linkage });
+        @export(__extendsftf2, .{ .name = "__extendsfkf2", .linkage = linkage });
+        @export(__extenddftf2, .{ .name = "__extenddfkf2", .linkage = linkage });
+        @export(__trunctfsf2, .{ .name = "__trunckfsf2", .linkage = linkage });
+        @export(__trunctfdf2, .{ .name = "__trunckfdf2", .linkage = linkage });
+        @export(__fixtfdi, .{ .name = "__fixkfdi", .linkage = linkage });
+        @export(__fixtfsi, .{ .name = "__fixkfsi", .linkage = linkage });
+        @export(__fixunstfsi, .{ .name = "__fixunskfsi", .linkage = linkage });
+        @export(__fixunstfdi, .{ .name = "__fixunskfdi", .linkage = linkage });
+        @export(__floatsitf, .{ .name = "__floatsikf", .linkage = linkage });
+        @export(__floatditf, .{ .name = "__floatdikf", .linkage = linkage });
+        @export(__floatunditf, .{ .name = "__floatundikf", .linkage = linkage });
+        @export(__floatunsitf, .{ .name = "__floatunsikf", .linkage = linkage });
+
+        @export(__letf2, .{ .name = "__eqkf2", .linkage = linkage });
+        @export(__letf2, .{ .name = "__nekf2", .linkage = linkage });
+        @export(__getf2, .{ .name = "__gekf2", .linkage = linkage });
+        @export(__letf2, .{ .name = "__ltkf2", .linkage = linkage });
+        @export(__letf2, .{ .name = "__lekf2", .linkage = linkage });
+        @export(__getf2, .{ .name = "__gtkf2", .linkage = linkage });
+        @export(__unordtf2, .{ .name = "__unordkf2", .linkage = linkage });
+    }
 
+    @export(floorf, .{ .name = "floorf", .linkage = linkage });
+    @export(floor, .{ .name = "floor", .linkage = linkage });
+    @export(floorl, .{ .name = "floorl", .linkage = linkage });
+
+    if (!builtin.zig_is_stage2) {
         @export(fmaq, .{ .name = "fmaq", .linkage = linkage });
-        @export(floorf, .{ .name = "floorf", .linkage = linkage });
-        @export(floor, .{ .name = "floor", .linkage = linkage });
-        @export(floorl, .{ .name = "floorl", .linkage = linkage });
     }
 }
 
diff --git a/lib/std/special/compiler_rt/absv.zig b/lib/std/special/compiler_rt/absv.zig
index 5c46d028a9c9..f14497daf204 100644
--- a/lib/std/special/compiler_rt/absv.zig
+++ b/lib/std/special/compiler_rt/absv.zig
@@ -2,31 +2,36 @@
 // * @panic, if value can not be represented
 // - absvXi4_generic for unoptimized version
 
-fn absvXi_generic(comptime ST: type) fn (a: ST) callconv(.C) ST {
-    return struct {
-        fn f(a: ST) callconv(.C) ST {
-            const UT = switch (ST) {
-                i32 => u32,
-                i64 => u64,
-                i128 => u128,
-                else => unreachable,
-            };
-            // taken from  Bit Twiddling Hacks
-            // compute the integer absolute value (abs) without branching
-            var x: ST = a;
-            const N: UT = @bitSizeOf(ST);
-            const sign: ST = a >> N - 1;
-            x +%= sign;
-            x ^= sign;
-            if (x < 0)
-                @panic("compiler_rt absv: overflow");
-            return x;
-        }
-    }.f;
+inline fn absvXi(comptime ST: type, a: ST) ST {
+    const UT = switch (ST) {
+        i32 => u32,
+        i64 => u64,
+        i128 => u128,
+        else => unreachable,
+    };
+    // taken from  Bit Twiddling Hacks
+    // compute the integer absolute value (abs) without branching
+    var x: ST = a;
+    const N: UT = @bitSizeOf(ST);
+    const sign: ST = a >> N - 1;
+    x +%= sign;
+    x ^= sign;
+    if (x < 0)
+        @panic("compiler_rt absv: overflow");
+    return x;
+}
+
+pub fn __absvsi2(a: i32) callconv(.C) i32 {
+    return absvXi(i32, a);
+}
+
+pub fn __absvdi2(a: i64) callconv(.C) i64 {
+    return absvXi(i64, a);
+}
+
+pub fn __absvti2(a: i128) callconv(.C) i128 {
+    return absvXi(i128, a);
 }
-pub const __absvsi2 = absvXi_generic(i32);
-pub const __absvdi2 = absvXi_generic(i64);
-pub const __absvti2 = absvXi_generic(i128);
 
 test {
     _ = @import("absvsi2_test.zig");
diff --git a/lib/std/special/compiler_rt/atomics.zig b/lib/std/special/compiler_rt/atomics.zig
index 3d93dc33b1c4..9b91fae600d4 100644
--- a/lib/std/special/compiler_rt/atomics.zig
+++ b/lib/std/special/compiler_rt/atomics.zig
@@ -131,213 +131,303 @@ comptime {
 // Specialized versions of the GCC atomic builtin functions.
 // LLVM emits those iff the object size is known and the pointers are correctly
 // aligned.
+inline fn atomic_load_N(comptime T: type, src: *T, model: i32) T {
+    _ = model;
+    if (@sizeOf(T) > largest_atomic_size) {
+        var sl = spinlocks.get(@ptrToInt(src));
+        defer sl.release();
+        return src.*;
+    } else {
+        return @atomicLoad(T, src, .SeqCst);
+    }
+}
 
-fn atomicLoadFn(comptime T: type) fn (*T, i32) callconv(.C) T {
-    return struct {
-        fn atomic_load_N(src: *T, model: i32) callconv(.C) T {
-            _ = model;
-            if (@sizeOf(T) > largest_atomic_size) {
-                var sl = spinlocks.get(@ptrToInt(src));
-                defer sl.release();
-                return src.*;
-            } else {
-                return @atomicLoad(T, src, .SeqCst);
-            }
-        }
-    }.atomic_load_N;
+fn __atomic_load_1(src: *u8, model: i32) callconv(.C) u8 {
+    return atomic_load_N(u8, src, model);
 }
 
-comptime {
-    if (supports_atomic_ops) {
-        const atomicLoad_u8 = atomicLoadFn(u8);
-        const atomicLoad_u16 = atomicLoadFn(u16);
-        const atomicLoad_u32 = atomicLoadFn(u32);
-        const atomicLoad_u64 = atomicLoadFn(u64);
-        @export(atomicLoad_u8, .{ .name = "__atomic_load_1", .linkage = linkage });
-        @export(atomicLoad_u16, .{ .name = "__atomic_load_2", .linkage = linkage });
-        @export(atomicLoad_u32, .{ .name = "__atomic_load_4", .linkage = linkage });
-        @export(atomicLoad_u64, .{ .name = "__atomic_load_8", .linkage = linkage });
-    }
+fn __atomic_load_2(src: *u16, model: i32) callconv(.C) u16 {
+    return atomic_load_N(u16, src, model);
 }
 
-fn atomicStoreFn(comptime T: type) fn (*T, T, i32) callconv(.C) void {
-    return struct {
-        fn atomic_store_N(dst: *T, value: T, model: i32) callconv(.C) void {
-            _ = model;
-            if (@sizeOf(T) > largest_atomic_size) {
-                var sl = spinlocks.get(@ptrToInt(dst));
-                defer sl.release();
-                dst.* = value;
-            } else {
-                @atomicStore(T, dst, value, .SeqCst);
-            }
-        }
-    }.atomic_store_N;
+fn __atomic_load_4(src: *u32, model: i32) callconv(.C) u32 {
+    return atomic_load_N(u32, src, model);
 }
 
-comptime {
-    if (supports_atomic_ops) {
-        const atomicStore_u8 = atomicStoreFn(u8);
-        const atomicStore_u16 = atomicStoreFn(u16);
-        const atomicStore_u32 = atomicStoreFn(u32);
-        const atomicStore_u64 = atomicStoreFn(u64);
-        @export(atomicStore_u8, .{ .name = "__atomic_store_1", .linkage = linkage });
-        @export(atomicStore_u16, .{ .name = "__atomic_store_2", .linkage = linkage });
-        @export(atomicStore_u32, .{ .name = "__atomic_store_4", .linkage = linkage });
-        @export(atomicStore_u64, .{ .name = "__atomic_store_8", .linkage = linkage });
+fn __atomic_load_8(src: *u64, model: i32) callconv(.C) u64 {
+    return atomic_load_N(u64, src, model);
+}
+
+inline fn atomic_store_N(comptime T: type, dst: *T, value: T, model: i32) void {
+    _ = model;
+    if (@sizeOf(T) > largest_atomic_size) {
+        var sl = spinlocks.get(@ptrToInt(dst));
+        defer sl.release();
+        dst.* = value;
+    } else {
+        @atomicStore(T, dst, value, .SeqCst);
     }
 }
 
-fn atomicExchangeFn(comptime T: type) fn (*T, T, i32) callconv(.C) T {
-    return struct {
-        fn atomic_exchange_N(ptr: *T, val: T, model: i32) callconv(.C) T {
-            _ = model;
-            if (@sizeOf(T) > largest_atomic_size) {
-                var sl = spinlocks.get(@ptrToInt(ptr));
-                defer sl.release();
-                const value = ptr.*;
-                ptr.* = val;
-                return value;
-            } else {
-                return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);
-            }
-        }
-    }.atomic_exchange_N;
+fn __atomic_store_1(dst: *u8, value: u8, model: i32) callconv(.C) void {
+    return atomic_store_N(u8, dst, value, model);
 }
 
-comptime {
-    if (supports_atomic_ops) {
-        const atomicExchange_u8 = atomicExchangeFn(u8);
-        const atomicExchange_u16 = atomicExchangeFn(u16);
-        const atomicExchange_u32 = atomicExchangeFn(u32);
-        const atomicExchange_u64 = atomicExchangeFn(u64);
-        @export(atomicExchange_u8, .{ .name = "__atomic_exchange_1", .linkage = linkage });
-        @export(atomicExchange_u16, .{ .name = "__atomic_exchange_2", .linkage = linkage });
-        @export(atomicExchange_u32, .{ .name = "__atomic_exchange_4", .linkage = linkage });
-        @export(atomicExchange_u64, .{ .name = "__atomic_exchange_8", .linkage = linkage });
+fn __atomic_store_2(dst: *u16, value: u16, model: i32) callconv(.C) void {
+    return atomic_store_N(u16, dst, value, model);
+}
+
+fn __atomic_store_4(dst: *u32, value: u32, model: i32) callconv(.C) void {
+    return atomic_store_N(u32, dst, value, model);
+}
+
+fn __atomic_store_8(dst: *u64, value: u64, model: i32) callconv(.C) void {
+    return atomic_store_N(u64, dst, value, model);
+}
+
+inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {
+    _ = model;
+    if (@sizeOf(T) > largest_atomic_size) {
+        var sl = spinlocks.get(@ptrToInt(ptr));
+        defer sl.release();
+        const value = ptr.*;
+        ptr.* = val;
+        return value;
+    } else {
+        return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);
     }
 }
 
-fn atomicCompareExchangeFn(comptime T: type) fn (*T, *T, T, i32, i32) callconv(.C) i32 {
-    return struct {
-        fn atomic_compare_exchange_N(ptr: *T, expected: *T, desired: T, success: i32, failure: i32) callconv(.C) i32 {
-            _ = success;
-            _ = failure;
-            if (@sizeOf(T) > largest_atomic_size) {
-                var sl = spinlocks.get(@ptrToInt(ptr));
-                defer sl.release();
-                const value = ptr.*;
-                if (value == expected.*) {
-                    ptr.* = desired;
-                    return 1;
-                }
-                expected.* = value;
-                return 0;
-            } else {
-                if (@cmpxchgStrong(T, ptr, expected.*, desired, .SeqCst, .SeqCst)) |old_value| {
-                    expected.* = old_value;
-                    return 0;
-                }
-                return 1;
-            }
+fn __atomic_exchange_1(ptr: *u8, val: u8, model: i32) callconv(.C) u8 {
+    return atomic_exchange_N(u8, ptr, val, model);
+}
+
+fn __atomic_exchange_2(ptr: *u16, val: u16, model: i32) callconv(.C) u16 {
+    return atomic_exchange_N(u16, ptr, val, model);
+}
+
+fn __atomic_exchange_4(ptr: *u32, val: u32, model: i32) callconv(.C) u32 {
+    return atomic_exchange_N(u32, ptr, val, model);
+}
+
+fn __atomic_exchange_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
+    return atomic_exchange_N(u64, ptr, val, model);
+}
+
+inline fn atomic_compare_exchange_N(
+    comptime T: type,
+    ptr: *T,
+    expected: *T,
+    desired: T,
+    success: i32,
+    failure: i32,
+) i32 {
+    _ = success;
+    _ = failure;
+    if (@sizeOf(T) > largest_atomic_size) {
+        var sl = spinlocks.get(@ptrToInt(ptr));
+        defer sl.release();
+        const value = ptr.*;
+        if (value == expected.*) {
+            ptr.* = desired;
+            return 1;
         }
-    }.atomic_compare_exchange_N;
+        expected.* = value;
+        return 0;
+    } else {
+        if (@cmpxchgStrong(T, ptr, expected.*, desired, .SeqCst, .SeqCst)) |old_value| {
+            expected.* = old_value;
+            return 0;
+        }
+        return 1;
+    }
 }
 
-comptime {
-    if (supports_atomic_ops) {
-        const atomicCompareExchange_u8 = atomicCompareExchangeFn(u8);
-        const atomicCompareExchange_u16 = atomicCompareExchangeFn(u16);
-        const atomicCompareExchange_u32 = atomicCompareExchangeFn(u32);
-        const atomicCompareExchange_u64 = atomicCompareExchangeFn(u64);
-        @export(atomicCompareExchange_u8, .{ .name = "__atomic_compare_exchange_1", .linkage = linkage });
-        @export(atomicCompareExchange_u16, .{ .name = "__atomic_compare_exchange_2", .linkage = linkage });
-        @export(atomicCompareExchange_u32, .{ .name = "__atomic_compare_exchange_4", .linkage = linkage });
-        @export(atomicCompareExchange_u64, .{ .name = "__atomic_compare_exchange_8", .linkage = linkage });
+fn __atomic_compare_exchange_1(ptr: *u8, expected: *u8, desired: u8, success: i32, failure: i32) callconv(.C) i32 {
+    return atomic_compare_exchange_N(u8, ptr, expected, desired, success, failure);
+}
+
+fn __atomic_compare_exchange_2(ptr: *u16, expected: *u16, desired: u16, success: i32, failure: i32) callconv(.C) i32 {
+    return atomic_compare_exchange_N(u16, ptr, expected, desired, success, failure);
+}
+
+fn __atomic_compare_exchange_4(ptr: *u32, expected: *u32, desired: u32, success: i32, failure: i32) callconv(.C) i32 {
+    return atomic_compare_exchange_N(u32, ptr, expected, desired, success, failure);
+}
+
+fn __atomic_compare_exchange_8(ptr: *u64, expected: *u64, desired: u64, success: i32, failure: i32) callconv(.C) i32 {
+    return atomic_compare_exchange_N(u64, ptr, expected, desired, success, failure);
+}
+
+inline fn fetch_op_N(comptime T: type, comptime op: std.builtin.AtomicRmwOp, ptr: *T, val: T, model: i32) T {
+    _ = model;
+    if (@sizeOf(T) > largest_atomic_size) {
+        var sl = spinlocks.get(@ptrToInt(ptr));
+        defer sl.release();
+
+        const value = ptr.*;
+        ptr.* = switch (op) {
+            .Add => value +% val,
+            .Sub => value -% val,
+            .And => value & val,
+            .Nand => ~(value & val),
+            .Or => value | val,
+            .Xor => value ^ val,
+            else => @compileError("unsupported atomic op"),
+        };
+
+        return value;
     }
+
+    return @atomicRmw(T, ptr, op, val, .SeqCst);
 }
 
-fn fetchFn(comptime T: type, comptime op: std.builtin.AtomicRmwOp) fn (*T, T, i32) callconv(.C) T {
-    return struct {
-        pub fn fetch_op_N(ptr: *T, val: T, model: i32) callconv(.C) T {
-            _ = model;
-            if (@sizeOf(T) > largest_atomic_size) {
-                var sl = spinlocks.get(@ptrToInt(ptr));
-                defer sl.release();
-
-                const value = ptr.*;
-                ptr.* = switch (op) {
-                    .Add => value +% val,
-                    .Sub => value -% val,
-                    .And => value & val,
-                    .Nand => ~(value & val),
-                    .Or => value | val,
-                    .Xor => value ^ val,
-                    else => @compileError("unsupported atomic op"),
-                };
-
-                return value;
-            }
+fn __atomic_fetch_add_1(ptr: *u8, val: u8, model: i32) callconv(.C) u8 {
+    return fetch_op_N(u8, .Add, ptr, val, model);
+}
 
-            return @atomicRmw(T, ptr, op, val, .SeqCst);
-        }
-    }.fetch_op_N;
+fn __atomic_fetch_add_2(ptr: *u16, val: u16, model: i32) callconv(.C) u16 {
+    return fetch_op_N(u16, .Add, ptr, val, model);
+}
+
+fn __atomic_fetch_add_4(ptr: *u32, val: u32, model: i32) callconv(.C) u32 {
+    return fetch_op_N(u32, .Add, ptr, val, model);
+}
+
+fn __atomic_fetch_add_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
+    return fetch_op_N(u64, .Add, ptr, val, model);
+}
+
+fn __atomic_fetch_sub_1(ptr: *u8, val: u8, model: i32) callconv(.C) u8 {
+    return fetch_op_N(u8, .Sub, ptr, val, model);
+}
+
+fn __atomic_fetch_sub_2(ptr: *u16, val: u16, model: i32) callconv(.C) u16 {
+    return fetch_op_N(u16, .Sub, ptr, val, model);
+}
+
+fn __atomic_fetch_sub_4(ptr: *u32, val: u32, model: i32) callconv(.C) u32 {
+    return fetch_op_N(u32, .Sub, ptr, val, model);
+}
+
+fn __atomic_fetch_sub_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
+    return fetch_op_N(u64, .Sub, ptr, val, model);
+}
+
+fn __atomic_fetch_and_1(ptr: *u8, val: u8, model: i32) callconv(.C) u8 {
+    return fetch_op_N(u8, .And, ptr, val, model);
+}
+
+fn __atomic_fetch_and_2(ptr: *u16, val: u16, model: i32) callconv(.C) u16 {
+    return fetch_op_N(u16, .And, ptr, val, model);
+}
+
+fn __atomic_fetch_and_4(ptr: *u32, val: u32, model: i32) callconv(.C) u32 {
+    return fetch_op_N(u32, .And, ptr, val, model);
+}
+
+fn __atomic_fetch_and_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
+    return fetch_op_N(u64, .And, ptr, val, model);
+}
+
+fn __atomic_fetch_or_1(ptr: *u8, val: u8, model: i32) callconv(.C) u8 {
+    return fetch_op_N(u8, .Or, ptr, val, model);
+}
+
+fn __atomic_fetch_or_2(ptr: *u16, val: u16, model: i32) callconv(.C) u16 {
+    return fetch_op_N(u16, .Or, ptr, val, model);
+}
+
+fn __atomic_fetch_or_4(ptr: *u32, val: u32, model: i32) callconv(.C) u32 {
+    return fetch_op_N(u32, .Or, ptr, val, model);
+}
+
+fn __atomic_fetch_or_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
+    return fetch_op_N(u64, .Or, ptr, val, model);
+}
+
+fn __atomic_fetch_xor_1(ptr: *u8, val: u8, model: i32) callconv(.C) u8 {
+    return fetch_op_N(u8, .Xor, ptr, val, model);
+}
+
+fn __atomic_fetch_xor_2(ptr: *u16, val: u16, model: i32) callconv(.C) u16 {
+    return fetch_op_N(u16, .Xor, ptr, val, model);
+}
+
+fn __atomic_fetch_xor_4(ptr: *u32, val: u32, model: i32) callconv(.C) u32 {
+    return fetch_op_N(u32, .Xor, ptr, val, model);
+}
+
+fn __atomic_fetch_xor_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
+    return fetch_op_N(u64, .Xor, ptr, val, model);
+}
+
+fn __atomic_fetch_nand_1(ptr: *u8, val: u8, model: i32) callconv(.C) u8 {
+    return fetch_op_N(u8, .Nand, ptr, val, model);
+}
+
+fn __atomic_fetch_nand_2(ptr: *u16, val: u16, model: i32) callconv(.C) u16 {
+    return fetch_op_N(u16, .Nand, ptr, val, model);
+}
+
+fn __atomic_fetch_nand_4(ptr: *u32, val: u32, model: i32) callconv(.C) u32 {
+    return fetch_op_N(u32, .Nand, ptr, val, model);
+}
+
+fn __atomic_fetch_nand_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
+    return fetch_op_N(u64, .Nand, ptr, val, model);
 }
 
 comptime {
     if (supports_atomic_ops) {
-        const fetch_add_u8 = fetchFn(u8, .Add);
-        const fetch_add_u16 = fetchFn(u16, .Add);
-        const fetch_add_u32 = fetchFn(u32, .Add);
-        const fetch_add_u64 = fetchFn(u64, .Add);
-        @export(fetch_add_u8, .{ .name = "__atomic_fetch_add_1", .linkage = linkage });
-        @export(fetch_add_u16, .{ .name = "__atomic_fetch_add_2", .linkage = linkage });
-        @export(fetch_add_u32, .{ .name = "__atomic_fetch_add_4", .linkage = linkage });
-        @export(fetch_add_u64, .{ .name = "__atomic_fetch_add_8", .linkage = linkage });
-
-        const fetch_sub_u8 = fetchFn(u8, .Sub);
-        const fetch_sub_u16 = fetchFn(u16, .Sub);
-        const fetch_sub_u32 = fetchFn(u32, .Sub);
-        const fetch_sub_u64 = fetchFn(u64, .Sub);
-        @export(fetch_sub_u8, .{ .name = "__atomic_fetch_sub_1", .linkage = linkage });
-        @export(fetch_sub_u16, .{ .name = "__atomic_fetch_sub_2", .linkage = linkage });
-        @export(fetch_sub_u32, .{ .name = "__atomic_fetch_sub_4", .linkage = linkage });
-        @export(fetch_sub_u64, .{ .name = "__atomic_fetch_sub_8", .linkage = linkage });
-
-        const fetch_and_u8 = fetchFn(u8, .And);
-        const fetch_and_u16 = fetchFn(u16, .And);
-        const fetch_and_u32 = fetchFn(u32, .And);
-        const fetch_and_u64 = fetchFn(u64, .And);
-        @export(fetch_and_u8, .{ .name = "__atomic_fetch_and_1", .linkage = linkage });
-        @export(fetch_and_u16, .{ .name = "__atomic_fetch_and_2", .linkage = linkage });
-        @export(fetch_and_u32, .{ .name = "__atomic_fetch_and_4", .linkage = linkage });
-        @export(fetch_and_u64, .{ .name = "__atomic_fetch_and_8", .linkage = linkage });
-
-        const fetch_or_u8 = fetchFn(u8, .Or);
-        const fetch_or_u16 = fetchFn(u16, .Or);
-        const fetch_or_u32 = fetchFn(u32, .Or);
-        const fetch_or_u64 = fetchFn(u64, .Or);
-        @export(fetch_or_u8, .{ .name = "__atomic_fetch_or_1", .linkage = linkage });
-        @export(fetch_or_u16, .{ .name = "__atomic_fetch_or_2", .linkage = linkage });
-        @export(fetch_or_u32, .{ .name = "__atomic_fetch_or_4", .linkage = linkage });
-        @export(fetch_or_u64, .{ .name = "__atomic_fetch_or_8", .linkage = linkage });
-
-        const fetch_xor_u8 = fetchFn(u8, .Xor);
-        const fetch_xor_u16 = fetchFn(u16, .Xor);
-        const fetch_xor_u32 = fetchFn(u32, .Xor);
-        const fetch_xor_u64 = fetchFn(u64, .Xor);
-        @export(fetch_xor_u8, .{ .name = "__atomic_fetch_xor_1", .linkage = linkage });
-        @export(fetch_xor_u16, .{ .name = "__atomic_fetch_xor_2", .linkage = linkage });
-        @export(fetch_xor_u32, .{ .name = "__atomic_fetch_xor_4", .linkage = linkage });
-        @export(fetch_xor_u64, .{ .name = "__atomic_fetch_xor_8", .linkage = linkage });
-
-        const fetch_nand_u8 = fetchFn(u8, .Nand);
-        const fetch_nand_u16 = fetchFn(u16, .Nand);
-        const fetch_nand_u32 = fetchFn(u32, .Nand);
-        const fetch_nand_u64 = fetchFn(u64, .Nand);
-        @export(fetch_nand_u8, .{ .name = "__atomic_fetch_nand_1", .linkage = linkage });
-        @export(fetch_nand_u16, .{ .name = "__atomic_fetch_nand_2", .linkage = linkage });
-        @export(fetch_nand_u32, .{ .name = "__atomic_fetch_nand_4", .linkage = linkage });
-        @export(fetch_nand_u64, .{ .name = "__atomic_fetch_nand_8", .linkage = linkage });
+        @export(__atomic_fetch_add_1, .{ .name = "__atomic_fetch_add_1", .linkage = linkage });
+        @export(__atomic_fetch_add_2, .{ .name = "__atomic_fetch_add_2", .linkage = linkage });
+        @export(__atomic_fetch_add_4, .{ .name = "__atomic_fetch_add_4", .linkage = linkage });
+        @export(__atomic_fetch_add_8, .{ .name = "__atomic_fetch_add_8", .linkage = linkage });
+
+        @export(__atomic_fetch_sub_1, .{ .name = "__atomic_fetch_sub_1", .linkage = linkage });
+        @export(__atomic_fetch_sub_2, .{ .name = "__atomic_fetch_sub_2", .linkage = linkage });
+        @export(__atomic_fetch_sub_4, .{ .name = "__atomic_fetch_sub_4", .linkage = linkage });
+        @export(__atomic_fetch_sub_8, .{ .name = "__atomic_fetch_sub_8", .linkage = linkage });
+
+        @export(__atomic_fetch_and_1, .{ .name = "__atomic_fetch_and_1", .linkage = linkage });
+        @export(__atomic_fetch_and_2, .{ .name = "__atomic_fetch_and_2", .linkage = linkage });
+        @export(__atomic_fetch_and_4, .{ .name = "__atomic_fetch_and_4", .linkage = linkage });
+        @export(__atomic_fetch_and_8, .{ .name = "__atomic_fetch_and_8", .linkage = linkage });
+
+        @export(__atomic_fetch_or_1, .{ .name = "__atomic_fetch_or_1", .linkage = linkage });
+        @export(__atomic_fetch_or_2, .{ .name = "__atomic_fetch_or_2", .linkage = linkage });
+        @export(__atomic_fetch_or_4, .{ .name = "__atomic_fetch_or_4", .linkage = linkage });
+        @export(__atomic_fetch_or_8, .{ .name = "__atomic_fetch_or_8", .linkage = linkage });
+
+        @export(__atomic_fetch_xor_1, .{ .name = "__atomic_fetch_xor_1", .linkage = linkage });
+        @export(__atomic_fetch_xor_2, .{ .name = "__atomic_fetch_xor_2", .linkage = linkage });
+        @export(__atomic_fetch_xor_4, .{ .name = "__atomic_fetch_xor_4", .linkage = linkage });
+        @export(__atomic_fetch_xor_8, .{ .name = "__atomic_fetch_xor_8", .linkage = linkage });
+
+        @export(__atomic_fetch_nand_1, .{ .name = "__atomic_fetch_nand_1", .linkage = linkage });
+        @export(__atomic_fetch_nand_2, .{ .name = "__atomic_fetch_nand_2", .linkage = linkage });
+        @export(__atomic_fetch_nand_4, .{ .name = "__atomic_fetch_nand_4", .linkage = linkage });
+        @export(__atomic_fetch_nand_8, .{ .name = "__atomic_fetch_nand_8", .linkage = linkage });
+
+        @export(__atomic_load_1, .{ .name = "__atomic_load_1", .linkage = linkage });
+        @export(__atomic_load_2, .{ .name = "__atomic_load_2", .linkage = linkage });
+        @export(__atomic_load_4, .{ .name = "__atomic_load_4", .linkage = linkage });
+        @export(__atomic_load_8, .{ .name = "__atomic_load_8", .linkage = linkage });
+
+        @export(__atomic_store_1, .{ .name = "__atomic_store_1", .linkage = linkage });
+        @export(__atomic_store_2, .{ .name = "__atomic_store_2", .linkage = linkage });
+        @export(__atomic_store_4, .{ .name = "__atomic_store_4", .linkage = linkage });
+        @export(__atomic_store_8, .{ .name = "__atomic_store_8", .linkage = linkage });
+
+        @export(__atomic_exchange_1, .{ .name = "__atomic_exchange_1", .linkage = linkage });
+        @export(__atomic_exchange_2, .{ .name = "__atomic_exchange_2", .linkage = linkage });
+        @export(__atomic_exchange_4, .{ .name = "__atomic_exchange_4", .linkage = linkage });
+        @export(__atomic_exchange_8, .{ .name = "__atomic_exchange_8", .linkage = linkage });
+
+        @export(__atomic_compare_exchange_1, .{ .name = "__atomic_compare_exchange_1", .linkage = linkage });
+        @export(__atomic_compare_exchange_2, .{ .name = "__atomic_compare_exchange_2", .linkage = linkage });
+        @export(__atomic_compare_exchange_4, .{ .name = "__atomic_compare_exchange_4", .linkage = linkage });
+        @export(__atomic_compare_exchange_8, .{ .name = "__atomic_compare_exchange_8", .linkage = linkage });
     }
 }
diff --git a/lib/std/special/compiler_rt/bswap.zig b/lib/std/special/compiler_rt/bswap.zig
index 0646f68c1a3c..f1d2138811e0 100644
--- a/lib/std/special/compiler_rt/bswap.zig
+++ b/lib/std/special/compiler_rt/bswap.zig
@@ -2,7 +2,7 @@ const std = @import("std");
 const builtin = @import("builtin");
 
 // bswap - byteswap
-// - bswapXi2_generic for unoptimized big and little endian
+// - bswapXi2 for unoptimized big and little endian
 // ie for u32
 // DE AD BE EF   <- little|big endian
 // FE BE AD DE   <- big|little endian
@@ -11,64 +11,64 @@ const builtin = @import("builtin");
 // 00 00 ff 00 << 1*8 (2n right  byte)
 // 00 00 00 ff << 3*8 (rightmost byte)
 
-fn bswapXi2_generic(comptime T: type) fn (a: T) callconv(.C) T {
-    return struct {
-        fn f(a: T) callconv(.C) T {
-            @setRuntimeSafety(builtin.is_test);
-            switch (@bitSizeOf(T)) {
-                32 => {
-                    // zig fmt: off
-                    return (((a & 0xff000000) >> 24)
-                         |  ((a & 0x00ff0000) >> 8 )
-                         |  ((a & 0x0000ff00) << 8 )
-                         |  ((a & 0x000000ff) << 24));
-                    // zig fmt: on
-                },
-                64 => {
-                    // zig fmt: off
-                    return (((a & 0xff00000000000000) >> 56)
-                         |  ((a & 0x00ff000000000000) >> 40 )
-                         |  ((a & 0x0000ff0000000000) >> 24 )
-                         |  ((a & 0x000000ff00000000) >> 8 )
-                         |  ((a & 0x00000000ff000000) << 8 )
-                         |  ((a & 0x0000000000ff0000) << 24 )
-                         |  ((a & 0x000000000000ff00) << 40 )
-                         |  ((a & 0x00000000000000ff) << 56));
-                    // zig fmt: on
-                },
-                128 => {
-                    // zig fmt: off
-                    return (((a & 0xff000000000000000000000000000000) >> 120)
-                         |  ((a & 0x00ff0000000000000000000000000000) >> 104)
-                         |  ((a & 0x0000ff00000000000000000000000000) >> 88 )
-                         |  ((a & 0x000000ff000000000000000000000000) >> 72 )
-                         |  ((a & 0x00000000ff0000000000000000000000) >> 56 )
-                         |  ((a & 0x0000000000ff00000000000000000000) >> 40 )
-                         |  ((a & 0x000000000000ff000000000000000000) >> 24 )
-                         |  ((a & 0x00000000000000ff0000000000000000) >> 8  )
-                         |  ((a & 0x0000000000000000ff00000000000000) << 8  )
-                         |  ((a & 0x000000000000000000ff000000000000) << 24 )
-                         |  ((a & 0x00000000000000000000ff0000000000) << 40 )
-                         |  ((a & 0x0000000000000000000000ff00000000) << 56 )
-                         |  ((a & 0x000000000000000000000000ff000000) << 72 )
-                         |  ((a & 0x00000000000000000000000000ff0000) << 88 )
-                         |  ((a & 0x0000000000000000000000000000ff00) << 104)
-                         |  ((a & 0x000000000000000000000000000000ff) << 120));
-                    // zig fmt: on
-                },
-                else => {
-                    unreachable;
-                },
-            }
-        }
-    }.f;
+inline fn bswapXi2(comptime T: type, a: T) T {
+    @setRuntimeSafety(builtin.is_test);
+    switch (@bitSizeOf(T)) {
+        32 => {
+            // zig fmt: off
+            return (((a & 0xff000000) >> 24)
+                 |  ((a & 0x00ff0000) >> 8 )
+                 |  ((a & 0x0000ff00) << 8 )
+                 |  ((a & 0x000000ff) << 24));
+            // zig fmt: on
+        },
+        64 => {
+            // zig fmt: off
+            return (((a & 0xff00000000000000) >> 56)
+                 |  ((a & 0x00ff000000000000) >> 40 )
+                 |  ((a & 0x0000ff0000000000) >> 24 )
+                 |  ((a & 0x000000ff00000000) >> 8 )
+                 |  ((a & 0x00000000ff000000) << 8 )
+                 |  ((a & 0x0000000000ff0000) << 24 )
+                 |  ((a & 0x000000000000ff00) << 40 )
+                 |  ((a & 0x00000000000000ff) << 56));
+            // zig fmt: on
+        },
+        128 => {
+            // zig fmt: off
+            return (((a & 0xff000000000000000000000000000000) >> 120)
+                 |  ((a & 0x00ff0000000000000000000000000000) >> 104)
+                 |  ((a & 0x0000ff00000000000000000000000000) >> 88 )
+                 |  ((a & 0x000000ff000000000000000000000000) >> 72 )
+                 |  ((a & 0x00000000ff0000000000000000000000) >> 56 )
+                 |  ((a & 0x0000000000ff00000000000000000000) >> 40 )
+                 |  ((a & 0x000000000000ff000000000000000000) >> 24 )
+                 |  ((a & 0x00000000000000ff0000000000000000) >> 8  )
+                 |  ((a & 0x0000000000000000ff00000000000000) << 8  )
+                 |  ((a & 0x000000000000000000ff000000000000) << 24 )
+                 |  ((a & 0x00000000000000000000ff0000000000) << 40 )
+                 |  ((a & 0x0000000000000000000000ff00000000) << 56 )
+                 |  ((a & 0x000000000000000000000000ff000000) << 72 )
+                 |  ((a & 0x00000000000000000000000000ff0000) << 88 )
+                 |  ((a & 0x0000000000000000000000000000ff00) << 104)
+                 |  ((a & 0x000000000000000000000000000000ff) << 120));
+            // zig fmt: on
+        },
+        else => unreachable,
+    }
 }
 
-pub const __bswapsi2 = bswapXi2_generic(u32);
+pub fn __bswapsi2(a: u32) callconv(.C) u32 {
+    return bswapXi2(u32, a);
+}
 
-pub const __bswapdi2 = bswapXi2_generic(u64);
+pub fn __bswapdi2(a: u64) callconv(.C) u64 {
+    return bswapXi2(u64, a);
+}
 
-pub const __bswapti2 = bswapXi2_generic(u128);
+pub fn __bswapti2(a: u128) callconv(.C) u128 {
+    return bswapXi2(u128, a);
+}
 
 test {
     _ = @import("bswapsi2_test.zig");
diff --git a/lib/std/special/compiler_rt/cmp.zig b/lib/std/special/compiler_rt/cmp.zig
index 630948ba2832..9eb4227527f2 100644
--- a/lib/std/special/compiler_rt/cmp.zig
+++ b/lib/std/special/compiler_rt/cmp.zig
@@ -11,28 +11,40 @@ const builtin = @import("builtin");
 // a == b => 1
 // a >  b => 2
 
-fn XcmpXi2_generic(comptime T: type) fn (a: T, b: T) callconv(.C) i32 {
-    return struct {
-        fn f(a: T, b: T) callconv(.C) i32 {
-            @setRuntimeSafety(builtin.is_test);
-            var cmp1: i32 = 0;
-            var cmp2: i32 = 0;
-            if (a > b)
-                cmp1 = 1;
-            if (a < b)
-                cmp2 = 1;
-            return cmp1 - cmp2 + 1;
-        }
-    }.f;
-}
-
-pub const __cmpsi2 = XcmpXi2_generic(i32);
-pub const __cmpdi2 = XcmpXi2_generic(i64);
-pub const __cmpti2 = XcmpXi2_generic(i128);
-
-pub const __ucmpsi2 = XcmpXi2_generic(u32);
-pub const __ucmpdi2 = XcmpXi2_generic(u64);
-pub const __ucmpti2 = XcmpXi2_generic(u128);
+inline fn XcmpXi2(comptime T: type, a: T, b: T) i32 {
+    @setRuntimeSafety(builtin.is_test);
+    var cmp1: i32 = 0;
+    var cmp2: i32 = 0;
+    if (a > b)
+        cmp1 = 1;
+    if (a < b)
+        cmp2 = 1;
+    return cmp1 - cmp2 + 1;
+}
+
+pub fn __cmpsi2(a: i32, b: i32) callconv(.C) i32 {
+    return XcmpXi2(i32, a, b);
+}
+
+pub fn __cmpdi2(a: i64, b: i64) callconv(.C) i32 {
+    return XcmpXi2(i64, a, b);
+}
+
+pub fn __cmpti2(a: i128, b: i128) callconv(.C) i32 {
+    return XcmpXi2(i128, a, b);
+}
+
+pub fn __ucmpsi2(a: u32, b: u32) callconv(.C) i32 {
+    return XcmpXi2(u32, a, b);
+}
+
+pub fn __ucmpdi2(a: u64, b: u64) callconv(.C) i32 {
+    return XcmpXi2(u64, a, b);
+}
+
+pub fn __ucmpti2(a: u128, b: u128) callconv(.C) i32 {
+    return XcmpXi2(u128, a, b);
+}
 
 test {
     _ = @import("cmpsi2_test.zig");
diff --git a/lib/std/special/compiler_rt/count0bits.zig b/lib/std/special/compiler_rt/count0bits.zig
index 29ba800b43bc..1f6d28ae0b06 100644
--- a/lib/std/special/compiler_rt/count0bits.zig
+++ b/lib/std/special/compiler_rt/count0bits.zig
@@ -2,44 +2,40 @@ const std = @import("std");
 const builtin = @import("builtin");
 
 // clz - count leading zeroes
-// - clzXi2_generic for unoptimized little and big endian
+// - clzXi2 for unoptimized little and big endian
 // - __clzsi2_thumb1: assume a != 0
 // - __clzsi2_arm32: assume a != 0
 
 // ctz - count trailing zeroes
-// - ctzXi2_generic for unoptimized little and big endian
+// - ctzXi2 for unoptimized little and big endian
 
 // ffs - find first set
 // * ffs = (a == 0) => 0, (a != 0) => ctz + 1
 // * dont pay for `if (x == 0) return shift;` inside ctz
-// - ffsXi2_generic for unoptimized little and big endian
-
-fn clzXi2_generic(comptime T: type) fn (a: T) callconv(.C) i32 {
-    return struct {
-        fn f(a: T) callconv(.C) i32 {
-            @setRuntimeSafety(builtin.is_test);
-
-            var x = switch (@bitSizeOf(T)) {
-                32 => @bitCast(u32, a),
-                64 => @bitCast(u64, a),
-                128 => @bitCast(u128, a),
-                else => unreachable,
-            };
-            var n: T = @bitSizeOf(T);
-            // Count first bit set using binary search, from Hacker's Delight
-            var y: @TypeOf(x) = 0;
-            comptime var shift: u8 = @bitSizeOf(T);
-            inline while (shift > 0) {
-                shift = shift >> 1;
-                y = x >> shift;
-                if (y != 0) {
-                    n = n - shift;
-                    x = y;
-                }
-            }
-            return @intCast(i32, n - @bitCast(T, x));
+// - ffsXi2 for unoptimized little and big endian
+
+inline fn clzXi2(comptime T: type, a: T) i32 {
+    @setRuntimeSafety(builtin.is_test);
+
+    var x = switch (@bitSizeOf(T)) {
+        32 => @bitCast(u32, a),
+        64 => @bitCast(u64, a),
+        128 => @bitCast(u128, a),
+        else => unreachable,
+    };
+    var n: T = @bitSizeOf(T);
+    // Count first bit set using binary search, from Hacker's Delight
+    var y: @TypeOf(x) = 0;
+    comptime var shift: u8 = @bitSizeOf(T);
+    inline while (shift > 0) {
+        shift = shift >> 1;
+        y = x >> shift;
+        if (y != 0) {
+            n = n - shift;
+            x = y;
         }
-    }.f;
+    }
+    return @intCast(i32, n - @bitCast(T, x));
 }
 
 fn __clzsi2_thumb1() callconv(.Naked) void {
@@ -125,103 +121,113 @@ fn __clzsi2_arm32() callconv(.Naked) void {
     unreachable;
 }
 
-pub const __clzsi2 = impl: {
-    switch (builtin.cpu.arch) {
-        .arm, .armeb, .thumb, .thumbeb => {
-            const use_thumb1 =
-                (builtin.cpu.arch.isThumb() or
-                std.Target.arm.featureSetHas(builtin.cpu.features, .noarm)) and
-                !std.Target.arm.featureSetHas(builtin.cpu.features, .thumb2);
-
-            if (use_thumb1) {
-                break :impl __clzsi2_thumb1;
-            }
-            // From here on we're either targeting Thumb2 or ARM.
-            else if (!builtin.cpu.arch.isThumb()) {
-                break :impl __clzsi2_arm32;
-            }
-            // Use the generic implementation otherwise.
-            else break :impl clzXi2_generic(i32);
-        },
-        else => break :impl clzXi2_generic(i32),
-    }
+fn clzsi2_generic(a: i32) callconv(.C) i32 {
+    return clzXi2(i32, a);
+}
+
+pub const __clzsi2 = switch (builtin.cpu.arch) {
+    .arm, .armeb, .thumb, .thumbeb => impl: {
+        const use_thumb1 =
+            (builtin.cpu.arch.isThumb() or
+            std.Target.arm.featureSetHas(builtin.cpu.features, .noarm)) and
+            !std.Target.arm.featureSetHas(builtin.cpu.features, .thumb2);
+
+        if (use_thumb1) {
+            break :impl __clzsi2_thumb1;
+        }
+        // From here on we're either targeting Thumb2 or ARM.
+        else if (!builtin.cpu.arch.isThumb()) {
+            break :impl __clzsi2_arm32;
+        }
+        // Use the generic implementation otherwise.
+        else break :impl clzsi2_generic;
+    },
+    else => clzsi2_generic,
 };
 
-pub const __clzdi2 = clzXi2_generic(i64);
-
-pub const __clzti2 = clzXi2_generic(i128);
-
-fn ctzXi2_generic(comptime T: type) fn (a: T) callconv(.C) i32 {
-    return struct {
-        fn f(a: T) callconv(.C) i32 {
-            @setRuntimeSafety(builtin.is_test);
-
-            var x = switch (@bitSizeOf(T)) {
-                32 => @bitCast(u32, a),
-                64 => @bitCast(u64, a),
-                128 => @bitCast(u128, a),
-                else => unreachable,
-            };
-            var n: T = 1;
-            // Number of trailing zeroes as binary search, from Hacker's Delight
-            var mask: @TypeOf(x) = std.math.maxInt(@TypeOf(x));
-            comptime var shift = @bitSizeOf(T);
-            if (x == 0) return shift;
-            inline while (shift > 1) {
-                shift = shift >> 1;
-                mask = mask >> shift;
-                if ((x & mask) == 0) {
-                    n = n + shift;
-                    x = x >> shift;
-                }
-            }
-            return @intCast(i32, n - @bitCast(T, (x & 1)));
+pub fn __clzdi2(a: i64) callconv(.C) i32 {
+    return clzXi2(i64, a);
+}
+
+pub fn __clzti2(a: i128) callconv(.C) i32 {
+    return clzXi2(i128, a);
+}
+
+inline fn ctzXi2(comptime T: type, a: T) i32 {
+    @setRuntimeSafety(builtin.is_test);
+
+    var x = switch (@bitSizeOf(T)) {
+        32 => @bitCast(u32, a),
+        64 => @bitCast(u64, a),
+        128 => @bitCast(u128, a),
+        else => unreachable,
+    };
+    var n: T = 1;
+    // Number of trailing zeroes as binary search, from Hacker's Delight
+    var mask: @TypeOf(x) = std.math.maxInt(@TypeOf(x));
+    comptime var shift = @bitSizeOf(T);
+    if (x == 0) return shift;
+    inline while (shift > 1) {
+        shift = shift >> 1;
+        mask = mask >> shift;
+        if ((x & mask) == 0) {
+            n = n + shift;
+            x = x >> shift;
         }
-    }.f;
+    }
+    return @intCast(i32, n - @bitCast(T, (x & 1)));
+}
+
+pub fn __ctzsi2(a: i32) callconv(.C) i32 {
+    return ctzXi2(i32, a);
 }
 
-pub const __ctzsi2 = ctzXi2_generic(i32);
-
-pub const __ctzdi2 = ctzXi2_generic(i64);
-
-pub const __ctzti2 = ctzXi2_generic(i128);
-
-fn ffsXi2_generic(comptime T: type) fn (a: T) callconv(.C) i32 {
-    return struct {
-        fn f(a: T) callconv(.C) i32 {
-            @setRuntimeSafety(builtin.is_test);
-
-            var x = switch (@bitSizeOf(T)) {
-                32 => @bitCast(u32, a),
-                64 => @bitCast(u64, a),
-                128 => @bitCast(u128, a),
-                else => unreachable,
-            };
-            var n: T = 1;
-            // adapted from Number of trailing zeroes (see ctzXi2_generic)
-            var mask: @TypeOf(x) = std.math.maxInt(@TypeOf(x));
-            comptime var shift = @bitSizeOf(T);
-            // In contrast to ctz return 0
-            if (x == 0) return 0;
-            inline while (shift > 1) {
-                shift = shift >> 1;
-                mask = mask >> shift;
-                if ((x & mask) == 0) {
-                    n = n + shift;
-                    x = x >> shift;
-                }
-            }
-            // return ctz + 1
-            return @intCast(i32, n - @bitCast(T, (x & 1))) + @as(i32, 1);
+pub fn __ctzdi2(a: i64) callconv(.C) i32 {
+    return ctzXi2(i64, a);
+}
+
+pub fn __ctzti2(a: i128) callconv(.C) i32 {
+    return ctzXi2(i128, a);
+}
+
+inline fn ffsXi2(comptime T: type, a: T) i32 {
+    @setRuntimeSafety(builtin.is_test);
+
+    var x = switch (@bitSizeOf(T)) {
+        32 => @bitCast(u32, a),
+        64 => @bitCast(u64, a),
+        128 => @bitCast(u128, a),
+        else => unreachable,
+    };
+    var n: T = 1;
+    // adapted from Number of trailing zeroes (see ctzXi2)
+    var mask: @TypeOf(x) = std.math.maxInt(@TypeOf(x));
+    comptime var shift = @bitSizeOf(T);
+    // In contrast to ctz return 0
+    if (x == 0) return 0;
+    inline while (shift > 1) {
+        shift = shift >> 1;
+        mask = mask >> shift;
+        if ((x & mask) == 0) {
+            n = n + shift;
+            x = x >> shift;
         }
-    }.f;
+    }
+    // return ctz + 1
+    return @intCast(i32, n - @bitCast(T, (x & 1))) + @as(i32, 1);
 }
 
-pub const __ffssi2 = ffsXi2_generic(i32);
+pub fn __ffssi2(a: i32) callconv(.C) i32 {
+    return ffsXi2(i32, a);
+}
 
-pub const __ffsdi2 = ffsXi2_generic(i64);
+pub fn __ffsdi2(a: i64) callconv(.C) i32 {
+    return ffsXi2(i64, a);
+}
 
-pub const __ffsti2 = ffsXi2_generic(i128);
+pub fn __ffsti2(a: i128) callconv(.C) i32 {
+    return ffsXi2(i128, a);
+}
 
 test {
     _ = @import("clzsi2_test.zig");
diff --git a/lib/std/special/compiler_rt/fixuint.zig b/lib/std/special/compiler_rt/fixuint.zig
index c51b80fbdb0b..6bfbcf6d654c 100644
--- a/lib/std/special/compiler_rt/fixuint.zig
+++ b/lib/std/special/compiler_rt/fixuint.zig
@@ -1,7 +1,7 @@
 const is_test = @import("builtin").is_test;
 const Log2Int = @import("std").math.Log2Int;
 
-pub fn fixuint(comptime fp_t: type, comptime fixuint_t: type, a: fp_t) fixuint_t {
+pub inline fn fixuint(comptime fp_t: type, comptime fixuint_t: type, a: fp_t) fixuint_t {
     @setRuntimeSafety(is_test);
 
     const rep_t = switch (fp_t) {
diff --git a/lib/std/special/compiler_rt/floatXisf.zig b/lib/std/special/compiler_rt/floatXisf.zig
index 9fc4c711001b..de3f4495cb07 100644
--- a/lib/std/special/compiler_rt/floatXisf.zig
+++ b/lib/std/special/compiler_rt/floatXisf.zig
@@ -4,7 +4,7 @@ const maxInt = std.math.maxInt;
 
 const FLT_MANT_DIG = 24;
 
-fn __floatXisf(comptime T: type, arg: T) f32 {
+inline fn floatXisf(comptime T: type, arg: T) f32 {
     @setRuntimeSafety(builtin.is_test);
 
     const bits = @typeInfo(T).Int.bits;
@@ -71,18 +71,15 @@ fn __floatXisf(comptime T: type, arg: T) f32 {
 }
 
 pub fn __floatdisf(arg: i64) callconv(.C) f32 {
-    @setRuntimeSafety(builtin.is_test);
-    return @call(.{ .modifier = .always_inline }, __floatXisf, .{ i64, arg });
+    return floatXisf(i64, arg);
 }
 
 pub fn __floattisf(arg: i128) callconv(.C) f32 {
-    @setRuntimeSafety(builtin.is_test);
-    return @call(.{ .modifier = .always_inline }, __floatXisf, .{ i128, arg });
+    return floatXisf(i128, arg);
 }
 
 pub fn __aeabi_l2f(arg: i64) callconv(.AAPCS) f32 {
-    @setRuntimeSafety(false);
-    return @call(.{ .modifier = .always_inline }, __floatdisf, .{arg});
+    return floatXisf(i64, arg);
 }
 
 test {
diff --git a/lib/std/special/compiler_rt/floatsiXf.zig b/lib/std/special/compiler_rt/floatsiXf.zig
index 23d5cb1e3c15..ef551d1911d4 100644
--- a/lib/std/special/compiler_rt/floatsiXf.zig
+++ b/lib/std/special/compiler_rt/floatsiXf.zig
@@ -2,7 +2,7 @@ const builtin = @import("builtin");
 const std = @import("std");
 const maxInt = std.math.maxInt;
 
-fn floatsiXf(comptime T: type, a: i32) T {
+inline fn floatsiXf(comptime T: type, a: i32) T {
     @setRuntimeSafety(builtin.is_test);
 
     const bits = @typeInfo(T).Float.bits;
@@ -56,27 +56,27 @@ fn floatsiXf(comptime T: type, a: i32) T {
 
 pub fn __floatsisf(arg: i32) callconv(.C) f32 {
     @setRuntimeSafety(builtin.is_test);
-    return @call(.{ .modifier = .always_inline }, floatsiXf, .{ f32, arg });
+    return floatsiXf(f32, arg);
 }
 
 pub fn __floatsidf(arg: i32) callconv(.C) f64 {
     @setRuntimeSafety(builtin.is_test);
-    return @call(.{ .modifier = .always_inline }, floatsiXf, .{ f64, arg });
+    return floatsiXf(f64, arg);
 }
 
 pub fn __floatsitf(arg: i32) callconv(.C) f128 {
     @setRuntimeSafety(builtin.is_test);
-    return @call(.{ .modifier = .always_inline }, floatsiXf, .{ f128, arg });
+    return floatsiXf(f128, arg);
 }
 
 pub fn __aeabi_i2d(arg: i32) callconv(.AAPCS) f64 {
     @setRuntimeSafety(false);
-    return @call(.{ .modifier = .always_inline }, __floatsidf, .{arg});
+    return floatsiXf(f64, arg);
 }
 
 pub fn __aeabi_i2f(arg: i32) callconv(.AAPCS) f32 {
     @setRuntimeSafety(false);
-    return @call(.{ .modifier = .always_inline }, __floatsisf, .{arg});
+    return floatsiXf(f32, arg);
 }
 
 fn test_one_floatsitf(a: i32, expected: u128) !void {
diff --git a/lib/std/special/compiler_rt/floatundisf.zig b/lib/std/special/compiler_rt/floatundisf.zig
index aca30ee30992..ffbe3ef252b6 100644
--- a/lib/std/special/compiler_rt/floatundisf.zig
+++ b/lib/std/special/compiler_rt/floatundisf.zig
@@ -4,7 +4,7 @@ const maxInt = std.math.maxInt;
 
 const FLT_MANT_DIG = 24;
 
-pub fn __floatundisf(arg: u64) callconv(.C) f32 {
+inline fn floatundisf(arg: u64) f32 {
     @setRuntimeSafety(builtin.is_test);
 
     if (arg == 0) return 0;
@@ -56,9 +56,12 @@ pub fn __floatundisf(arg: u64) callconv(.C) f32 {
     return @bitCast(f32, result);
 }
 
+pub fn __floatundisf(arg: u64) callconv(.C) f32 {
+    return floatundisf(arg);
+}
+
 pub fn __aeabi_ul2f(arg: u64) callconv(.AAPCS) f32 {
-    @setRuntimeSafety(false);
-    return @call(.{ .modifier = .always_inline }, __floatundisf, .{arg});
+    return floatundisf(arg);
 }
 
 fn test__floatundisf(a: u64, expected: f32) !void {
diff --git a/lib/std/special/compiler_rt/floatunsidf.zig b/lib/std/special/compiler_rt/floatunsidf.zig
index 555d4f5657a3..f474c1de8f73 100644
--- a/lib/std/special/compiler_rt/floatunsidf.zig
+++ b/lib/std/special/compiler_rt/floatunsidf.zig
@@ -4,7 +4,7 @@ const maxInt = std.math.maxInt;
 
 const implicitBit = @as(u64, 1) << 52;
 
-pub fn __floatunsidf(arg: u32) callconv(.C) f64 {
+inline fn floatunsidf(arg: u32) f64 {
     @setRuntimeSafety(builtin.is_test);
 
     if (arg == 0) return 0.0;
@@ -18,9 +18,12 @@ pub fn __floatunsidf(arg: u32) callconv(.C) f64 {
     return @bitCast(f64, mant | (exp + 1023) << 52);
 }
 
+pub fn __floatunsidf(arg: u32) callconv(.C) f64 {
+    return floatunsidf(arg);
+}
+
 pub fn __aeabi_ui2d(arg: u32) callconv(.AAPCS) f64 {
-    @setRuntimeSafety(false);
-    return @call(.{ .modifier = .always_inline }, __floatunsidf, .{arg});
+    return floatunsidf(arg);
 }
 
 fn test_one_floatunsidf(a: u32, expected: u64) !void {
diff --git a/lib/std/special/compiler_rt/floatunsisf.zig b/lib/std/special/compiler_rt/floatunsisf.zig
index c8a654ff7a7d..d267baee0152 100644
--- a/lib/std/special/compiler_rt/floatunsisf.zig
+++ b/lib/std/special/compiler_rt/floatunsisf.zig
@@ -6,7 +6,7 @@ const significandBits = 23;
 const exponentBias = 127;
 const implicitBit = @as(u32, 1) << significandBits;
 
-pub fn __floatunsisf(arg: u32) callconv(.C) f32 {
+inline fn floatunsisf(arg: u32) f32 {
     @setRuntimeSafety(builtin.is_test);
 
     if (arg == 0) return 0.0;
@@ -38,9 +38,12 @@ pub fn __floatunsisf(arg: u32) callconv(.C) f32 {
     return @bitCast(f32, result);
 }
 
+pub fn __floatunsisf(arg: u32) callconv(.C) f32 {
+    return floatunsisf(arg);
+}
+
 pub fn __aeabi_ui2f(arg: u32) callconv(.AAPCS) f32 {
-    @setRuntimeSafety(false);
-    return @call(.{ .modifier = .always_inline }, __floatunsisf, .{arg});
+    return floatunsisf(arg);
 }
 
 fn test_one_floatunsisf(a: u32, expected: u32) !void {
diff --git a/lib/std/special/compiler_rt/negXi2.zig b/lib/std/special/compiler_rt/negXi2.zig
index de89c8deb7aa..15102b5df7b1 100644
--- a/lib/std/special/compiler_rt/negXi2.zig
+++ b/lib/std/special/compiler_rt/negXi2.zig
@@ -2,7 +2,7 @@ const std = @import("std");
 const builtin = @import("builtin");
 
 // neg - negate (the number)
-// - negXi2_generic for unoptimized little and big endian
+// - negXi2 for unoptimized little and big endian
 
 // sfffffff = 2^31-1
 // two's complement inverting bits and add 1 would result in -INT_MIN == 0
@@ -11,20 +11,22 @@ const builtin = @import("builtin");
 // * size optimized builds
 // * machines that dont support carry operations
 
-fn negXi2_generic(comptime T: type) fn (a: T) callconv(.C) T {
-    return struct {
-        fn f(a: T) callconv(.C) T {
-            @setRuntimeSafety(builtin.is_test);
-            return -a;
-        }
-    }.f;
+inline fn negXi2(comptime T: type, a: T) T {
+    @setRuntimeSafety(builtin.is_test);
+    return -a;
 }
 
-pub const __negsi2 = negXi2_generic(i32);
+pub fn __negsi2(a: i32) callconv(.C) i32 {
+    return negXi2(i32, a);
+}
 
-pub const __negdi2 = negXi2_generic(i64);
+pub fn __negdi2(a: i64) callconv(.C) i64 {
+    return negXi2(i64, a);
+}
 
-pub const __negti2 = negXi2_generic(i128);
+pub fn __negti2(a: i128) callconv(.C) i128 {
+    return negXi2(i128, a);
+}
 
 test {
     _ = @import("negsi2_test.zig");
diff --git a/lib/std/special/compiler_rt/negv.zig b/lib/std/special/compiler_rt/negv.zig
index 99525a6e5be1..09abb040d5ec 100644
--- a/lib/std/special/compiler_rt/negv.zig
+++ b/lib/std/special/compiler_rt/negv.zig
@@ -3,26 +3,31 @@
 // - negvXi4_generic for unoptimized version
 
 // assume -0 == 0 is gracefully handled by the hardware
-fn negvXi_generic(comptime ST: type) fn (a: ST) callconv(.C) ST {
-    return struct {
-        fn f(a: ST) callconv(.C) ST {
-            const UT = switch (ST) {
-                i32 => u32,
-                i64 => u64,
-                i128 => u128,
-                else => unreachable,
-            };
-            const N: UT = @bitSizeOf(ST);
-            const min: ST = @bitCast(ST, (@as(UT, 1) << (N - 1)));
-            if (a == min)
-                @panic("compiler_rt negv: overflow");
-            return -a;
-        }
-    }.f;
+inline fn negvXi(comptime ST: type, a: ST) ST {
+    const UT = switch (ST) {
+        i32 => u32,
+        i64 => u64,
+        i128 => u128,
+        else => unreachable,
+    };
+    const N: UT = @bitSizeOf(ST);
+    const min: ST = @bitCast(ST, (@as(UT, 1) << (N - 1)));
+    if (a == min)
+        @panic("compiler_rt negv: overflow");
+    return -a;
+}
+
+pub fn __negvsi2(a: i32) callconv(.C) i32 {
+    return negvXi(i32, a);
+}
+
+pub fn __negvdi2(a: i64) callconv(.C) i64 {
+    return negvXi(i64, a);
+}
+
+pub fn __negvti2(a: i128) callconv(.C) i128 {
+    return negvXi(i128, a);
 }
-pub const __negvsi2 = negvXi_generic(i32);
-pub const __negvdi2 = negvXi_generic(i64);
-pub const __negvti2 = negvXi_generic(i128);
 
 test {
     _ = @import("negvsi2_test.zig");
diff --git a/lib/std/special/compiler_rt/parity.zig b/lib/std/special/compiler_rt/parity.zig
index 1c47aa3c73f0..ae634b079036 100644
--- a/lib/std/special/compiler_rt/parity.zig
+++ b/lib/std/special/compiler_rt/parity.zig
@@ -4,34 +4,36 @@ const builtin = @import("builtin");
 // parity - if number of bits set is even => 0, else => 1
 // - pariytXi2_generic for big and little endian
 
-fn parityXi2_generic(comptime T: type) fn (a: T) callconv(.C) i32 {
-    return struct {
-        fn f(a: T) callconv(.C) i32 {
-            @setRuntimeSafety(builtin.is_test);
+inline fn parityXi2(comptime T: type, a: T) i32 {
+    @setRuntimeSafety(builtin.is_test);
 
-            var x = switch (@bitSizeOf(T)) {
-                32 => @bitCast(u32, a),
-                64 => @bitCast(u64, a),
-                128 => @bitCast(u128, a),
-                else => unreachable,
-            };
-            // Bit Twiddling Hacks: Compute parity in parallel
-            comptime var shift: u8 = @bitSizeOf(T) / 2;
-            inline while (shift > 2) {
-                x ^= x >> shift;
-                shift = shift >> 1;
-            }
-            x &= 0xf;
-            return (@intCast(u16, 0x6996) >> @intCast(u4, x)) & 1; // optimization for >>2 and >>1
-        }
-    }.f;
+    var x = switch (@bitSizeOf(T)) {
+        32 => @bitCast(u32, a),
+        64 => @bitCast(u64, a),
+        128 => @bitCast(u128, a),
+        else => unreachable,
+    };
+    // Bit Twiddling Hacks: Compute parity in parallel
+    comptime var shift: u8 = @bitSizeOf(T) / 2;
+    inline while (shift > 2) {
+        x ^= x >> shift;
+        shift = shift >> 1;
+    }
+    x &= 0xf;
+    return (@intCast(u16, 0x6996) >> @intCast(u4, x)) & 1; // optimization for >>2 and >>1
 }
 
-pub const __paritysi2 = parityXi2_generic(i32);
+pub fn __paritysi2(a: i32) callconv(.C) i32 {
+    return parityXi2(i32, a);
+}
 
-pub const __paritydi2 = parityXi2_generic(i64);
+pub fn __paritydi2(a: i64) callconv(.C) i32 {
+    return parityXi2(i64, a);
+}
 
-pub const __parityti2 = parityXi2_generic(i128);
+pub fn __parityti2(a: i128) callconv(.C) i32 {
+    return parityXi2(i128, a);
+}
 
 test {
     _ = @import("paritysi2_test.zig");
diff --git a/lib/std/special/compiler_rt/popcount.zig b/lib/std/special/compiler_rt/popcount.zig
index 72513895db03..362b232fb854 100644
--- a/lib/std/special/compiler_rt/popcount.zig
+++ b/lib/std/special/compiler_rt/popcount.zig
@@ -10,35 +10,37 @@ const std = @import("std");
 // TAOCP: Combinational Algorithms, Bitwise Tricks And Techniques,
 //   subsubsection "Working with the rightmost bits" and "Sideways addition".
 
-fn popcountXi2_generic(comptime ST: type) fn (a: ST) callconv(.C) i32 {
-    return struct {
-        fn f(a: ST) callconv(.C) i32 {
-            @setRuntimeSafety(builtin.is_test);
-            const UT = switch (ST) {
-                i32 => u32,
-                i64 => u64,
-                i128 => u128,
-                else => unreachable,
-            };
-            var x = @bitCast(UT, a);
-            x -= (x >> 1) & (~@as(UT, 0) / 3); // 0x55...55, aggregate duos
-            x = ((x >> 2) & (~@as(UT, 0) / 5)) // 0x33...33, aggregate nibbles
-            + (x & (~@as(UT, 0) / 5));
-            x += x >> 4;
-            x &= ~@as(UT, 0) / 17; // 0x0F...0F, aggregate bytes
-            // 8 most significant bits of x + (x<<8) + (x<<16) + ..
-            x *%= ~@as(UT, 0) / 255; // 0x01...01
-            x >>= (@bitSizeOf(ST) - 8);
-            return @intCast(i32, x);
-        }
-    }.f;
+inline fn popcountXi2(comptime ST: type, a: ST) i32 {
+    @setRuntimeSafety(builtin.is_test);
+    const UT = switch (ST) {
+        i32 => u32,
+        i64 => u64,
+        i128 => u128,
+        else => unreachable,
+    };
+    var x = @bitCast(UT, a);
+    x -= (x >> 1) & (~@as(UT, 0) / 3); // 0x55...55, aggregate duos
+    x = ((x >> 2) & (~@as(UT, 0) / 5)) // 0x33...33, aggregate nibbles
+    + (x & (~@as(UT, 0) / 5));
+    x += x >> 4;
+    x &= ~@as(UT, 0) / 17; // 0x0F...0F, aggregate bytes
+    // 8 most significant bits of x + (x<<8) + (x<<16) + ..
+    x *%= ~@as(UT, 0) / 255; // 0x01...01
+    x >>= (@bitSizeOf(ST) - 8);
+    return @intCast(i32, x);
 }
 
-pub const __popcountsi2 = popcountXi2_generic(i32);
+pub fn __popcountsi2(a: i32) callconv(.C) i32 {
+    return popcountXi2(i32, a);
+}
 
-pub const __popcountdi2 = popcountXi2_generic(i64);
+pub fn __popcountdi2(a: i64) callconv(.C) i32 {
+    return popcountXi2(i64, a);
+}
 
-pub const __popcountti2 = popcountXi2_generic(i128);
+pub fn __popcountti2(a: i128) callconv(.C) i32 {
+    return popcountXi2(i128, a);
+}
 
 test {
     _ = @import("popcountsi2_test.zig");
diff --git a/lib/std/special/compiler_rt/shift.zig b/lib/std/special/compiler_rt/shift.zig
index b20516e46c14..edcf246daf75 100644
--- a/lib/std/special/compiler_rt/shift.zig
+++ b/lib/std/special/compiler_rt/shift.zig
@@ -19,7 +19,7 @@ fn Dwords(comptime T: type, comptime signed_half: bool) type {
 
 // Arithmetic shift left
 // Precondition: 0 <= b < bits_in_dword
-pub fn ashlXi3(comptime T: type, a: T, b: i32) T {
+pub inline fn ashlXi3(comptime T: type, a: T, b: i32) T {
     const dwords = Dwords(T, false);
     const S = Log2Int(dwords.HalfT);
 
@@ -42,7 +42,7 @@ pub fn ashlXi3(comptime T: type, a: T, b: i32) T {
 
 // Arithmetic shift right
 // Precondition: 0 <= b < T.bit_count
-pub fn ashrXi3(comptime T: type, a: T, b: i32) T {
+pub inline fn ashrXi3(comptime T: type, a: T, b: i32) T {
     const dwords = Dwords(T, true);
     const S = Log2Int(dwords.HalfT);
 
@@ -69,7 +69,7 @@ pub fn ashrXi3(comptime T: type, a: T, b: i32) T {
 
 // Logical shift right
 // Precondition: 0 <= b < T.bit_count
-pub fn lshrXi3(comptime T: type, a: T, b: i32) T {
+pub inline fn lshrXi3(comptime T: type, a: T, b: i32) T {
     const dwords = Dwords(T, false);
     const S = Log2Int(dwords.HalfT);
 
@@ -91,32 +91,32 @@ pub fn lshrXi3(comptime T: type, a: T, b: i32) T {
 }
 
 pub fn __ashldi3(a: i64, b: i32) callconv(.C) i64 {
-    return @call(.{ .modifier = .always_inline }, ashlXi3, .{ i64, a, b });
+    return ashlXi3(i64, a, b);
 }
 pub fn __ashlti3(a: i128, b: i32) callconv(.C) i128 {
-    return @call(.{ .modifier = .always_inline }, ashlXi3, .{ i128, a, b });
+    return ashlXi3(i128, a, b);
 }
 pub fn __ashrdi3(a: i64, b: i32) callconv(.C) i64 {
-    return @call(.{ .modifier = .always_inline }, ashrXi3, .{ i64, a, b });
+    return ashrXi3(i64, a, b);
 }
 pub fn __ashrti3(a: i128, b: i32) callconv(.C) i128 {
-    return @call(.{ .modifier = .always_inline }, ashrXi3, .{ i128, a, b });
+    return ashrXi3(i128, a, b);
 }
 pub fn __lshrdi3(a: i64, b: i32) callconv(.C) i64 {
-    return @call(.{ .modifier = .always_inline }, lshrXi3, .{ i64, a, b });
+    return lshrXi3(i64, a, b);
 }
 pub fn __lshrti3(a: i128, b: i32) callconv(.C) i128 {
-    return @call(.{ .modifier = .always_inline }, lshrXi3, .{ i128, a, b });
+    return lshrXi3(i128, a, b);
 }
 
 pub fn __aeabi_llsl(a: i64, b: i32) callconv(.AAPCS) i64 {
-    return __ashldi3(a, b);
+    return ashlXi3(i64, a, b);
 }
 pub fn __aeabi_lasr(a: i64, b: i32) callconv(.AAPCS) i64 {
-    return __ashrdi3(a, b);
+    return ashrXi3(i64, a, b);
 }
 pub fn __aeabi_llsr(a: i64, b: i32) callconv(.AAPCS) i64 {
-    return __lshrdi3(a, b);
+    return lshrXi3(i64, a, b);
 }
 
 test {
diff --git a/src/type.zig b/src/type.zig
index 5bcf310fc40e..9de6bb25a8c7 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -4603,7 +4603,18 @@ pub const CType = enum {
                 .longlong,
                 .ulonglong,
                 => return 64,
-                .longdouble => @panic("TODO figure out what kind of float `long double` is on this target"),
+                .longdouble => switch (target.cpu.arch) {
+                    .riscv64,
+                    .aarch64,
+                    .aarch64_be,
+                    .aarch64_32,
+                    .s390x,
+                    .mips64,
+                    .mips64el,
+                    => return 128,
+
+                    else => return 80,
+                },
             },
 
             .windows, .uefi => switch (self) {

From b4d6e85a339e971829404dc1a260bde4062735f8 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 2 Jan 2022 14:11:37 -0700
Subject: [PATCH 13/17] Sema: implement peer type resolution of signed and
 unsigned ints

This allows stage2 to build more of compiler-rt.

I also changed `-%` to `-` for comptime ints in the div and mul
implementations of compiler-rt. This is clearer code and also happens to
work around a bug in stage2.
---
 lib/std/special/compiler_rt.zig        | 66 ++++++++++++--------------
 lib/std/special/compiler_rt/divdf3.zig |  2 +-
 lib/std/special/compiler_rt/divsf3.zig |  2 +-
 lib/std/special/compiler_rt/divtf3.zig |  2 +-
 lib/std/special/compiler_rt/mulXf3.zig |  2 +-
 src/Sema.zig                           | 23 ++++-----
 src/type.zig                           |  4 +-
 test/behavior/cast.zig                 |  9 ++++
 test/behavior/cast_stage1.zig          |  9 ----
 9 files changed, 58 insertions(+), 61 deletions(-)

diff --git a/lib/std/special/compiler_rt.zig b/lib/std/special/compiler_rt.zig
index 29c6bdc7c353..78017726479f 100644
--- a/lib/std/special/compiler_rt.zig
+++ b/lib/std/special/compiler_rt.zig
@@ -221,21 +221,19 @@ comptime {
     const __subtf3 = @import("compiler_rt/addXf3.zig").__subtf3;
     @export(__subtf3, .{ .name = "__subtf3", .linkage = linkage });
 
-    if (!builtin.zig_is_stage2) {
-        const __mulsf3 = @import("compiler_rt/mulXf3.zig").__mulsf3;
-        @export(__mulsf3, .{ .name = "__mulsf3", .linkage = linkage });
-        const __muldf3 = @import("compiler_rt/mulXf3.zig").__muldf3;
-        @export(__muldf3, .{ .name = "__muldf3", .linkage = linkage });
-        const __multf3 = @import("compiler_rt/mulXf3.zig").__multf3;
-        @export(__multf3, .{ .name = "__multf3", .linkage = linkage });
-
-        const __divsf3 = @import("compiler_rt/divsf3.zig").__divsf3;
-        @export(__divsf3, .{ .name = "__divsf3", .linkage = linkage });
-        const __divdf3 = @import("compiler_rt/divdf3.zig").__divdf3;
-        @export(__divdf3, .{ .name = "__divdf3", .linkage = linkage });
-        const __divtf3 = @import("compiler_rt/divtf3.zig").__divtf3;
-        @export(__divtf3, .{ .name = "__divtf3", .linkage = linkage });
-    }
+    const __mulsf3 = @import("compiler_rt/mulXf3.zig").__mulsf3;
+    @export(__mulsf3, .{ .name = "__mulsf3", .linkage = linkage });
+    const __muldf3 = @import("compiler_rt/mulXf3.zig").__muldf3;
+    @export(__muldf3, .{ .name = "__muldf3", .linkage = linkage });
+    const __multf3 = @import("compiler_rt/mulXf3.zig").__multf3;
+    @export(__multf3, .{ .name = "__multf3", .linkage = linkage });
+
+    const __divsf3 = @import("compiler_rt/divsf3.zig").__divsf3;
+    @export(__divsf3, .{ .name = "__divsf3", .linkage = linkage });
+    const __divdf3 = @import("compiler_rt/divdf3.zig").__divdf3;
+    @export(__divdf3, .{ .name = "__divdf3", .linkage = linkage });
+    const __divtf3 = @import("compiler_rt/divtf3.zig").__divtf3;
+    @export(__divtf3, .{ .name = "__divtf3", .linkage = linkage });
 
     // Integral bit manipulation
     const __ashldi3 = @import("compiler_rt/shift.zig").__ashldi3;
@@ -251,26 +249,24 @@ comptime {
     const __lshrti3 = @import("compiler_rt/shift.zig").__lshrti3;
     @export(__lshrti3, .{ .name = "__lshrti3", .linkage = linkage });
 
-    if (!builtin.zig_is_stage2) {
-        const __clzsi2 = @import("compiler_rt/count0bits.zig").__clzsi2;
-        @export(__clzsi2, .{ .name = "__clzsi2", .linkage = linkage });
-        const __clzdi2 = @import("compiler_rt/count0bits.zig").__clzdi2;
-        @export(__clzdi2, .{ .name = "__clzdi2", .linkage = linkage });
-        const __clzti2 = @import("compiler_rt/count0bits.zig").__clzti2;
-        @export(__clzti2, .{ .name = "__clzti2", .linkage = linkage });
-        const __ctzsi2 = @import("compiler_rt/count0bits.zig").__ctzsi2;
-        @export(__ctzsi2, .{ .name = "__ctzsi2", .linkage = linkage });
-        const __ctzdi2 = @import("compiler_rt/count0bits.zig").__ctzdi2;
-        @export(__ctzdi2, .{ .name = "__ctzdi2", .linkage = linkage });
-        const __ctzti2 = @import("compiler_rt/count0bits.zig").__ctzti2;
-        @export(__ctzti2, .{ .name = "__ctzti2", .linkage = linkage });
-        const __ffssi2 = @import("compiler_rt/count0bits.zig").__ffssi2;
-        @export(__ffssi2, .{ .name = "__ffssi2", .linkage = linkage });
-        const __ffsdi2 = @import("compiler_rt/count0bits.zig").__ffsdi2;
-        @export(__ffsdi2, .{ .name = "__ffsdi2", .linkage = linkage });
-        const __ffsti2 = @import("compiler_rt/count0bits.zig").__ffsti2;
-        @export(__ffsti2, .{ .name = "__ffsti2", .linkage = linkage });
-    }
+    const __clzsi2 = @import("compiler_rt/count0bits.zig").__clzsi2;
+    @export(__clzsi2, .{ .name = "__clzsi2", .linkage = linkage });
+    const __clzdi2 = @import("compiler_rt/count0bits.zig").__clzdi2;
+    @export(__clzdi2, .{ .name = "__clzdi2", .linkage = linkage });
+    const __clzti2 = @import("compiler_rt/count0bits.zig").__clzti2;
+    @export(__clzti2, .{ .name = "__clzti2", .linkage = linkage });
+    const __ctzsi2 = @import("compiler_rt/count0bits.zig").__ctzsi2;
+    @export(__ctzsi2, .{ .name = "__ctzsi2", .linkage = linkage });
+    const __ctzdi2 = @import("compiler_rt/count0bits.zig").__ctzdi2;
+    @export(__ctzdi2, .{ .name = "__ctzdi2", .linkage = linkage });
+    const __ctzti2 = @import("compiler_rt/count0bits.zig").__ctzti2;
+    @export(__ctzti2, .{ .name = "__ctzti2", .linkage = linkage });
+    const __ffssi2 = @import("compiler_rt/count0bits.zig").__ffssi2;
+    @export(__ffssi2, .{ .name = "__ffssi2", .linkage = linkage });
+    const __ffsdi2 = @import("compiler_rt/count0bits.zig").__ffsdi2;
+    @export(__ffsdi2, .{ .name = "__ffsdi2", .linkage = linkage });
+    const __ffsti2 = @import("compiler_rt/count0bits.zig").__ffsti2;
+    @export(__ffsti2, .{ .name = "__ffsti2", .linkage = linkage });
 
     const __paritysi2 = @import("compiler_rt/parity.zig").__paritysi2;
     @export(__paritysi2, .{ .name = "__paritysi2", .linkage = linkage });
diff --git a/lib/std/special/compiler_rt/divdf3.zig b/lib/std/special/compiler_rt/divdf3.zig
index ebb139f32406..2148902de25e 100644
--- a/lib/std/special/compiler_rt/divdf3.zig
+++ b/lib/std/special/compiler_rt/divdf3.zig
@@ -35,7 +35,7 @@ pub fn __divdf3(a: f64, b: f64) callconv(.C) f64 {
     var scale: i32 = 0;
 
     // Detect if a or b is zero, denormal, infinity, or NaN.
-    if (aExponent -% 1 >= maxExponent -% 1 or bExponent -% 1 >= maxExponent -% 1) {
+    if (aExponent -% 1 >= maxExponent - 1 or bExponent -% 1 >= maxExponent - 1) {
         const aAbs: Z = @bitCast(Z, a) & absMask;
         const bAbs: Z = @bitCast(Z, b) & absMask;
 
diff --git a/lib/std/special/compiler_rt/divsf3.zig b/lib/std/special/compiler_rt/divsf3.zig
index af4cfaaa868e..5e7dc7bb44c8 100644
--- a/lib/std/special/compiler_rt/divsf3.zig
+++ b/lib/std/special/compiler_rt/divsf3.zig
@@ -34,7 +34,7 @@ pub fn __divsf3(a: f32, b: f32) callconv(.C) f32 {
     var scale: i32 = 0;
 
     // Detect if a or b is zero, denormal, infinity, or NaN.
-    if (aExponent -% 1 >= maxExponent -% 1 or bExponent -% 1 >= maxExponent -% 1) {
+    if (aExponent -% 1 >= maxExponent - 1 or bExponent -% 1 >= maxExponent - 1) {
         const aAbs: Z = @bitCast(Z, a) & absMask;
         const bAbs: Z = @bitCast(Z, b) & absMask;
 
diff --git a/lib/std/special/compiler_rt/divtf3.zig b/lib/std/special/compiler_rt/divtf3.zig
index 4dce86087dcf..fc26c6026657 100644
--- a/lib/std/special/compiler_rt/divtf3.zig
+++ b/lib/std/special/compiler_rt/divtf3.zig
@@ -33,7 +33,7 @@ pub fn __divtf3(a: f128, b: f128) callconv(.C) f128 {
     var scale: i32 = 0;
 
     // Detect if a or b is zero, denormal, infinity, or NaN.
-    if (aExponent -% 1 >= maxExponent -% 1 or bExponent -% 1 >= maxExponent -% 1) {
+    if (aExponent -% 1 >= maxExponent - 1 or bExponent -% 1 >= maxExponent - 1) {
         const aAbs: Z = @bitCast(Z, a) & absMask;
         const bAbs: Z = @bitCast(Z, b) & absMask;
 
diff --git a/lib/std/special/compiler_rt/mulXf3.zig b/lib/std/special/compiler_rt/mulXf3.zig
index 1e9171c8cb79..48c31f47b100 100644
--- a/lib/std/special/compiler_rt/mulXf3.zig
+++ b/lib/std/special/compiler_rt/mulXf3.zig
@@ -56,7 +56,7 @@ fn mulXf3(comptime T: type, a: T, b: T) T {
     var scale: i32 = 0;
 
     // Detect if a or b is zero, denormal, infinity, or NaN.
-    if (aExponent -% 1 >= maxExponent -% 1 or bExponent -% 1 >= maxExponent -% 1) {
+    if (aExponent -% 1 >= maxExponent - 1 or bExponent -% 1 >= maxExponent - 1) {
         const aAbs: Z = @bitCast(Z, a) & absMask;
         const bAbs: Z = @bitCast(Z, b) & absMask;
 
diff --git a/src/Sema.zig b/src/Sema.zig
index b65e88360f15..42bf795bd90d 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -14634,11 +14634,11 @@ fn resolvePeerTypes(
     instructions: []Air.Inst.Ref,
     candidate_srcs: Module.PeerTypeCandidateSrc,
 ) !Type {
-    if (instructions.len == 0)
-        return Type.initTag(.noreturn);
-
-    if (instructions.len == 1)
-        return sema.typeOf(instructions[0]);
+    switch (instructions.len) {
+        0 => return Type.initTag(.noreturn),
+        1 => return sema.typeOf(instructions[0]),
+        else => {},
+    }
 
     const target = sema.mod.getTarget();
 
@@ -14668,13 +14668,14 @@ fn resolvePeerTypes(
                     continue;
                 },
                 .Int => {
-                    if (chosen_ty.isSignedInt() == candidate_ty.isSignedInt()) {
-                        if (chosen_ty.intInfo(target).bits < candidate_ty.intInfo(target).bits) {
-                            chosen = candidate;
-                            chosen_i = candidate_i + 1;
-                        }
-                        continue;
+                    const chosen_info = chosen_ty.intInfo(target);
+                    const candidate_info = candidate_ty.intInfo(target);
+
+                    if (chosen_info.bits < candidate_info.bits) {
+                        chosen = candidate;
+                        chosen_i = candidate_i + 1;
                     }
+                    continue;
                 },
                 .Pointer => if (chosen_ty.ptrSize() == .C) continue,
                 else => {},
diff --git a/src/type.zig b/src/type.zig
index 9de6bb25a8c7..b94809399468 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -3106,9 +3106,9 @@ pub const Type = extern union {
             .c_ulonglong => return .{ .signedness = .unsigned, .bits = CType.ulonglong.sizeInBits(target) },
 
             .enum_full, .enum_nonexhaustive => ty = ty.cast(Payload.EnumFull).?.data.tag_ty,
-            .enum_numbered => ty = self.castTag(.enum_numbered).?.data.tag_ty,
+            .enum_numbered => ty = ty.castTag(.enum_numbered).?.data.tag_ty,
             .enum_simple => {
-                const enum_obj = self.castTag(.enum_simple).?.data;
+                const enum_obj = ty.castTag(.enum_simple).?.data;
                 const field_count = enum_obj.fields.count();
                 if (field_count == 0) return .{ .signedness = .unsigned, .bits = 0 };
                 return .{ .signedness = .unsigned, .bits = smallestUnsignedBits(field_count - 1) };
diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig
index c4ebbc7643e3..d1edefeb2d2c 100644
--- a/test/behavior/cast.zig
+++ b/test/behavior/cast.zig
@@ -295,3 +295,12 @@ test "cast from ?[*]T to ??[*]T" {
     const a: ??[*]u8 = @as(?[*]u8, null);
     try expect(a != null and a.? == null);
 }
+
+test "peer type unsigned int to signed" {
+    var w: u31 = 5;
+    var x: u8 = 7;
+    var y: i32 = -5;
+    var a = w + y + x;
+    comptime try expect(@TypeOf(a) == i32);
+    try expect(a == 7);
+}
diff --git a/test/behavior/cast_stage1.zig b/test/behavior/cast_stage1.zig
index a4854b30b280..ff4c657ed7c4 100644
--- a/test/behavior/cast_stage1.zig
+++ b/test/behavior/cast_stage1.zig
@@ -383,15 +383,6 @@ test "peer type resolve string lit with sentinel-terminated mutable slice" {
     comptime try expect(@TypeOf("hi", slice) == [:0]const u8);
 }
 
-test "peer type unsigned int to signed" {
-    var w: u31 = 5;
-    var x: u8 = 7;
-    var y: i32 = -5;
-    var a = w + y + x;
-    comptime try expect(@TypeOf(a) == i32);
-    try expect(a == 7);
-}
-
 test "peer type resolve array pointers, one of them const" {
     var array1: [4]u8 = undefined;
     const array2: [5]u8 = undefined;

From d3f87f8ac01039722197a13a12342fc747a90567 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Fri, 31 Dec 2021 16:35:01 -0800
Subject: [PATCH 14/17] std.fs.rename: fix Windows implementation

The semantics of this function are that it moves both files and
directories. Previously we had this `is_dir` boolean field of
`std.os.windows.OpenFile` which required the API user to choose: are we
opening a file or directory? And the other kind would either cause
error.IsDir or error.NotDir. But that is not a limitation of the Windows
file system API; it was self-imposed.

On Windows, rename is implemented internally with `NtCreateFile` so we
need to allow it to open either files or directories. This is now done
by `std.os.windows.OpenFile` accepting enum{file_only,dir_only,any}
instead of a boolean.
---
 lib/std/fs.zig         |  2 +-
 lib/std/fs/watch.zig   |  2 +-
 lib/std/os.zig         | 11 ++++++-----
 lib/std/os/windows.zig | 23 ++++++++++++++++++-----
 4 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/lib/std/fs.zig b/lib/std/fs.zig
index 6e1821178f39..a09f9f2ed2f2 100644
--- a/lib/std/fs.zig
+++ b/lib/std/fs.zig
@@ -1361,7 +1361,7 @@ pub const Dir = struct {
                     .share_access = share_access,
                     .creation = creation,
                     .io_mode = .blocking,
-                    .open_dir = true,
+                    .filter = .dir_only,
                 }) catch |er| switch (er) {
                     error.WouldBlock => unreachable,
                     else => |e2| return e2,
diff --git a/lib/std/fs/watch.zig b/lib/std/fs/watch.zig
index c103925bdd8c..e2ec8b8061a5 100644
--- a/lib/std/fs/watch.zig
+++ b/lib/std/fs/watch.zig
@@ -401,7 +401,7 @@ pub fn Watch(comptime V: type) type {
                     .access_mask = windows.FILE_LIST_DIRECTORY,
                     .creation = windows.FILE_OPEN,
                     .io_mode = .evented,
-                    .open_dir = true,
+                    .filter = .dir_only,
                 });
                 errdefer windows.CloseHandle(dir_handle);
 
diff --git a/lib/std/os.zig b/lib/std/os.zig
index 1728c2ac0d65..7be8825fcc3c 100644
--- a/lib/std/os.zig
+++ b/lib/std/os.zig
@@ -1353,7 +1353,7 @@ fn openOptionsFromFlags(flags: u32) windows.OpenFileOptions {
         access_mask |= w.GENERIC_READ | w.GENERIC_WRITE;
     }
 
-    const open_dir: bool = flags & O.DIRECTORY != 0;
+    const filter: windows.OpenFileOptions.Filter = if (flags & O.DIRECTORY != 0) .dir_only else .file_only;
     const follow_symlinks: bool = flags & O.NOFOLLOW == 0;
 
     const creation: w.ULONG = blk: {
@@ -1369,7 +1369,7 @@ fn openOptionsFromFlags(flags: u32) windows.OpenFileOptions {
         .access_mask = access_mask,
         .io_mode = .blocking,
         .creation = creation,
-        .open_dir = open_dir,
+        .filter = filter,
         .follow_symlinks = follow_symlinks,
     };
 }
@@ -2324,6 +2324,7 @@ pub fn renameatW(
         .access_mask = windows.SYNCHRONIZE | windows.GENERIC_WRITE | windows.DELETE,
         .creation = windows.FILE_OPEN,
         .io_mode = .blocking,
+        .filter = .any, // This function is supposed to rename both files and directories.
     }) catch |err| switch (err) {
         error.WouldBlock => unreachable, // Not possible without `.share_access_nonblocking = true`.
         else => |e| return e,
@@ -2435,7 +2436,7 @@ pub fn mkdiratW(dir_fd: fd_t, sub_path_w: []const u16, mode: u32) MakeDirError!v
         .access_mask = windows.GENERIC_READ | windows.SYNCHRONIZE,
         .creation = windows.FILE_CREATE,
         .io_mode = .blocking,
-        .open_dir = true,
+        .filter = .dir_only,
     }) catch |err| switch (err) {
         error.IsDir => unreachable,
         error.PipeBusy => unreachable,
@@ -2511,7 +2512,7 @@ pub fn mkdirW(dir_path_w: []const u16, mode: u32) MakeDirError!void {
         .access_mask = windows.GENERIC_READ | windows.SYNCHRONIZE,
         .creation = windows.FILE_CREATE,
         .io_mode = .blocking,
-        .open_dir = true,
+        .filter = .dir_only,
     }) catch |err| switch (err) {
         error.IsDir => unreachable,
         error.PipeBusy => unreachable,
@@ -4693,7 +4694,7 @@ pub fn realpathW(pathname: []const u16, out_buffer: *[MAX_PATH_BYTES]u8) RealPat
                 .share_access = share_access,
                 .creation = creation,
                 .io_mode = .blocking,
-                .open_dir = true,
+                .filter = .dir_only,
             }) catch |er| switch (er) {
                 error.WouldBlock => unreachable,
                 else => |e2| return e2,
diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index 59e65ed54c18..0d9907893cd5 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -53,17 +53,26 @@ pub const OpenFileOptions = struct {
     io_mode: std.io.ModeOverride,
     /// If true, tries to open path as a directory.
     /// Defaults to false.
-    open_dir: bool = false,
+    filter: Filter = .file_only,
     /// If false, tries to open path as a reparse point without dereferencing it.
     /// Defaults to true.
     follow_symlinks: bool = true,
+
+    pub const Filter = enum {
+        /// Causes `OpenFile` to return `error.IsDir` if the opened handle would be a directory.
+        file_only,
+        /// Causes `OpenFile` to return `error.NotDir` if the opened handle would be a file.
+        dir_only,
+        /// `OpenFile` does not discriminate between opening files and directories.
+        any,
+    };
 };
 
 pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
-    if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
+    if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and options.filter == .file_only) {
         return error.IsDir;
     }
-    if (mem.eql(u16, sub_path_w, &[_]u16{ '.', '.' }) and !options.open_dir) {
+    if (mem.eql(u16, sub_path_w, &[_]u16{ '.', '.' }) and options.filter == .file_only) {
         return error.IsDir;
     }
 
@@ -87,7 +96,11 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
     };
     var io: IO_STATUS_BLOCK = undefined;
     const blocking_flag: ULONG = if (options.io_mode == .blocking) FILE_SYNCHRONOUS_IO_NONALERT else 0;
-    const file_or_dir_flag: ULONG = if (options.open_dir) FILE_DIRECTORY_FILE else FILE_NON_DIRECTORY_FILE;
+    const file_or_dir_flag: ULONG = switch (options.filter) {
+        .file_only => FILE_NON_DIRECTORY_FILE,
+        .dir_only => FILE_DIRECTORY_FILE,
+        .any => 0,
+    };
     // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
     const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
 
@@ -695,7 +708,7 @@ pub fn CreateSymbolicLink(
         .dir = dir,
         .creation = FILE_CREATE,
         .io_mode = .blocking,
-        .open_dir = is_directory,
+        .filter = if (is_directory) .dir_only else .file_only,
     }) catch |err| switch (err) {
         error.IsDir => return error.PathAlreadyExists,
         error.NotDir => unreachable,

From 5e086b2b4c4558681cc23c6cc602704fd06d6fc8 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 2 Jan 2022 17:58:54 -0700
Subject: [PATCH 15/17] compiler-rt: small refactor in atomics

---
 lib/std/special/compiler_rt/atomics.zig | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/lib/std/special/compiler_rt/atomics.zig b/lib/std/special/compiler_rt/atomics.zig
index 9b91fae600d4..7727d7af3d9d 100644
--- a/lib/std/special/compiler_rt/atomics.zig
+++ b/lib/std/special/compiler_rt/atomics.zig
@@ -119,15 +119,6 @@ fn __atomic_compare_exchange(
     return 0;
 }
 
-comptime {
-    if (supports_atomic_ops) {
-        @export(__atomic_load, .{ .name = "__atomic_load", .linkage = linkage });
-        @export(__atomic_store, .{ .name = "__atomic_store", .linkage = linkage });
-        @export(__atomic_exchange, .{ .name = "__atomic_exchange", .linkage = linkage });
-        @export(__atomic_compare_exchange, .{ .name = "__atomic_compare_exchange", .linkage = linkage });
-    }
-}
-
 // Specialized versions of the GCC atomic builtin functions.
 // LLVM emits those iff the object size is known and the pointers are correctly
 // aligned.
@@ -380,6 +371,11 @@ fn __atomic_fetch_nand_8(ptr: *u64, val: u64, model: i32) callconv(.C) u64 {
 
 comptime {
     if (supports_atomic_ops) {
+        @export(__atomic_load, .{ .name = "__atomic_load", .linkage = linkage });
+        @export(__atomic_store, .{ .name = "__atomic_store", .linkage = linkage });
+        @export(__atomic_exchange, .{ .name = "__atomic_exchange", .linkage = linkage });
+        @export(__atomic_compare_exchange, .{ .name = "__atomic_compare_exchange", .linkage = linkage });
+
         @export(__atomic_fetch_add_1, .{ .name = "__atomic_fetch_add_1", .linkage = linkage });
         @export(__atomic_fetch_add_2, .{ .name = "__atomic_fetch_add_2", .linkage = linkage });
         @export(__atomic_fetch_add_4, .{ .name = "__atomic_fetch_add_4", .linkage = linkage });

From 663ffa5a7a1d53d1adf9c1ea6991aad8bf82aadc Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 2 Jan 2022 18:54:16 -0700
Subject: [PATCH 16/17] stage2: fix missing items from whole cache mode hash

Without this, Zig would re-use a compiler-rt built with stage2 when one
built by stage1 was needed.
---
 src/Compilation.zig | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 8b761cd45032..b01b4bbae423 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1263,6 +1263,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                     // track it and packages as files.
                 },
             }
+
+            // Synchronize with other matching comments: ZigOnlyHashStuff
             hash.add(valgrind);
             hash.add(single_threaded);
             hash.add(use_stage1);
@@ -1304,11 +1306,11 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             errdefer artifact_dir.close();
             const zig_cache_artifact_directory: Directory = .{
                 .handle = artifact_dir,
-                .path = if (options.local_cache_directory.path) |p|
-                    try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir })
-                else
-                    artifact_sub_dir,
+                .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}),
             };
+            log.debug("zig_cache_artifact_directory='{s}' use_stage1={}", .{
+                zig_cache_artifact_directory.path, use_stage1,
+            });
 
             const builtin_pkg = try Package.createWithDir(
                 gpa,
@@ -2220,6 +2222,18 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
             try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
         }
 
+        // Synchronize with other matching comments: ZigOnlyHashStuff
+        man.hash.add(comp.bin_file.options.valgrind);
+        man.hash.add(comp.bin_file.options.single_threaded);
+        man.hash.add(comp.bin_file.options.use_stage1);
+        man.hash.add(comp.bin_file.options.use_llvm);
+        man.hash.add(comp.bin_file.options.dll_export_fns);
+        man.hash.add(comp.bin_file.options.is_test);
+        man.hash.add(comp.test_evented_io);
+        man.hash.addOptionalBytes(comp.test_filter);
+        man.hash.addOptionalBytes(comp.test_name_prefix);
+        man.hash.add(comp.bin_file.options.skip_linker_dependencies);
+        man.hash.add(comp.bin_file.options.parent_compilation_link_libc);
         man.hash.add(mod.emit_h != null);
     }
 

From d94303be2bcee33e7efba22a186fd06eaa809707 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 2 Jan 2022 20:57:04 -0700
Subject: [PATCH 17/17] stage2: introduce renameTmpIntoCache into the linker
 API

Doc comments reproduced here:

This function is called by the frontend before flush(). It communicates that
`options.bin_file.emit` directory needs to be renamed from
`[zig-cache]/tmp/[random]` to `[zig-cache]/o/[digest]`.
The frontend would like to simply perform a file system rename, however,
some linker backends care about the file paths of the objects they are linking.
So this function call tells linker backends to rename the paths of object files
to observe the new directory path.
Linker backends which do not have this requirement can fall back to the simple
implementation at the bottom of this function.
This function is only called when CacheMode is `whole`.

This solves stack trace regressions on Windows and macOS because the
linker backends do not observe object file paths until flush().
---
 src/Compilation.zig  | 84 ++++++++++++++++++++++++++------------------
 src/codegen/llvm.zig | 10 +++---
 src/link.zig         | 59 +++++++++++++++++++++++++------
 src/link/Coff.zig    | 21 +++++++----
 src/link/Elf.zig     | 26 +++++++++-----
 src/link/MachO.zig   | 23 ++++++++----
 src/link/Wasm.zig    | 22 +++++++-----
 7 files changed, 166 insertions(+), 79 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index b01b4bbae423..cbb44c1e183a 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2093,39 +2093,15 @@ pub fn update(comp: *Compilation) !void {
     }
 
     if (comp.totalErrorCount() != 0) {
-        // Skip flushing.
+        // Skip flushing and keep source files loaded for error reporting.
         comp.link_error_flags = .{};
         return;
     }
 
-    // This is needed before reading the error flags.
-    try comp.bin_file.flush(comp);
-    comp.link_error_flags = comp.bin_file.errorFlags();
-
-    if (!use_stage1) {
-        if (comp.bin_file.options.module) |module| {
-            try link.File.C.flushEmitH(module);
-        }
-    }
-
     // Flush takes care of -femit-bin, but we still have -femit-llvm-ir, -femit-llvm-bc, and
     // -femit-asm to handle, in the case of C objects.
     comp.emitOthers();
 
-    // If there are any errors, we anticipate the source files being loaded
-    // to report error messages. Otherwise we unload all source files to save memory.
-    // The ZIR needs to stay loaded in memory because (1) Decl objects contain references
-    // to it, and (2) generic instantiations, comptime calls, inline calls will need
-    // to reference the ZIR.
-    if (!comp.keep_source_files_loaded) {
-        if (comp.bin_file.options.module) |module| {
-            for (module.import_table.values()) |file| {
-                file.unloadTree(comp.gpa);
-                file.unloadSource(comp.gpa);
-            }
-        }
-    }
-
     if (comp.whole_cache_manifest != null) {
         const digest = man.final();
 
@@ -2139,23 +2115,63 @@ pub fn update(comp: *Compilation) !void {
         const o_sub_path = try std.fs.path.join(comp.gpa, &[_][]const u8{ "o", &digest });
         defer comp.gpa.free(o_sub_path);
 
-        try std.fs.rename(
-            comp.local_cache_directory.handle,
-            tmp_dir_sub_path,
-            comp.local_cache_directory.handle,
-            o_sub_path,
-        );
+        try comp.bin_file.renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path);
+        comp.wholeCacheModeSetBinFilePath(&digest);
+
+        // This is intentionally sandwiched between renameTmpIntoCache() and writeManifest().
+        if (comp.bin_file.options.module) |module| {
+            // We need to set the zig_cache_artifact_directory for -femit-asm, -femit-llvm-ir,
+            // etc to know where to output to.
+            var artifact_dir = try comp.local_cache_directory.handle.openDir(o_sub_path, .{});
+            defer artifact_dir.close();
+
+            var dir_path = try comp.local_cache_directory.join(comp.gpa, &.{o_sub_path});
+            defer comp.gpa.free(dir_path);
+
+            module.zig_cache_artifact_directory = .{
+                .handle = artifact_dir,
+                .path = dir_path,
+            };
+
+            try comp.flush();
+        } else {
+            try comp.flush();
+        }
 
         // Failure here only means an unnecessary cache miss.
         man.writeManifest() catch |err| {
             log.warn("failed to write cache manifest: {s}", .{@errorName(err)});
         };
 
-        comp.wholeCacheModeSetBinFilePath(&digest);
-
         assert(comp.bin_file.lock == null);
         comp.bin_file.lock = man.toOwnedLock();
-        return;
+    } else {
+        try comp.flush();
+    }
+
+    // Unload all source files to save memory.
+    // The ZIR needs to stay loaded in memory because (1) Decl objects contain references
+    // to it, and (2) generic instantiations, comptime calls, inline calls will need
+    // to reference the ZIR.
+    if (!comp.keep_source_files_loaded) {
+        if (comp.bin_file.options.module) |module| {
+            for (module.import_table.values()) |file| {
+                file.unloadTree(comp.gpa);
+                file.unloadSource(comp.gpa);
+            }
+        }
+    }
+}
+
+fn flush(comp: *Compilation) !void {
+    try comp.bin_file.flush(comp); // This is needed before reading the error flags.
+    comp.link_error_flags = comp.bin_file.errorFlags();
+
+    const use_stage1 = build_options.is_stage1 and comp.bin_file.options.use_stage1;
+    if (!use_stage1) {
+        if (comp.bin_file.options.module) |module| {
+            try link.File.C.flushEmitH(module);
+        }
     }
 }
 
diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
index a248fb17188e..e29116476d06 100644
--- a/src/codegen/llvm.zig
+++ b/src/codegen/llvm.zig
@@ -324,10 +324,12 @@ pub const Object = struct {
         const mod = comp.bin_file.options.module.?;
         const cache_dir = mod.zig_cache_artifact_directory;
 
-        const emit_bin_path: ?[*:0]const u8 = if (comp.bin_file.options.emit) |emit|
-            try emit.directory.joinZ(arena, &[_][]const u8{self.sub_path})
-        else
-            null;
+        const emit_bin_path: ?[*:0]const u8 = if (comp.bin_file.options.emit) |emit| blk: {
+            const full_out_path = try emit.directory.join(arena, &[_][]const u8{emit.sub_path});
+            break :blk try std.fs.path.joinZ(arena, &.{
+                std.fs.path.dirname(full_out_path).?, self.sub_path,
+            });
+        } else null;
 
         const emit_asm_path = try locPath(arena, comp.emit_asm, cache_dir);
         const emit_llvm_ir_path = try locPath(arena, comp.emit_llvm_ir, cache_dir);
diff --git a/src/link.zig b/src/link.zig
index a720bdada2b9..5c33ce3b7008 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -636,6 +636,36 @@ pub const File = struct {
         }
     }
 
+    /// This function is called by the frontend before flush(). It communicates that
+    /// `options.bin_file.emit` directory needs to be renamed from
+    /// `[zig-cache]/tmp/[random]` to `[zig-cache]/o/[digest]`.
+    /// The frontend would like to simply perform a file system rename, however,
+    /// some linker backends care about the file paths of the objects they are linking.
+    /// So this function call tells linker backends to rename the paths of object files
+    /// to observe the new directory path.
+    /// Linker backends which do not have this requirement can fall back to the simple
+    /// implementation at the bottom of this function.
+    /// This function is only called when CacheMode is `whole`.
+    pub fn renameTmpIntoCache(
+        base: *File,
+        cache_directory: Compilation.Directory,
+        tmp_dir_sub_path: []const u8,
+        o_sub_path: []const u8,
+    ) !void {
+        // So far, none of the linker backends need to respond to this event, however,
+        // it makes sense that they might want to. So we leave this mechanism here
+        // for now. Once the linker backends get more mature, if it turns out this
+        // is not needed we can refactor this into having the frontend do the rename
+        // directly, and remove this function from link.zig.
+        _ = base;
+        try std.fs.rename(
+            cache_directory.handle,
+            tmp_dir_sub_path,
+            cache_directory.handle,
+            o_sub_path,
+        );
+    }
+
     pub fn linkAsArchive(base: *File, comp: *Compilation) !void {
         const tracy = trace(@src());
         defer tracy.end();
@@ -645,9 +675,11 @@ pub const File = struct {
         const arena = arena_allocator.allocator();
 
         const directory = base.options.emit.?.directory; // Just an alias to make it shorter to type.
+        const full_out_path = try directory.join(arena, &[_][]const u8{base.options.emit.?.sub_path});
+        const full_out_path_z = try arena.dupeZ(u8, full_out_path);
 
-        // If there is no Zig code to compile, then we should skip flushing the output file because it
-        // will not be part of the linker line anyway.
+        // If there is no Zig code to compile, then we should skip flushing the output file
+        // because it will not be part of the linker line anyway.
         const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: {
             const use_stage1 = build_options.is_stage1 and base.options.use_stage1;
             if (use_stage1) {
@@ -656,20 +688,28 @@ pub const File = struct {
                     .target = base.options.target,
                     .output_mode = .Obj,
                 });
-                const o_directory = module.zig_cache_artifact_directory;
-                const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
-                break :blk full_obj_path;
+                switch (base.options.cache_mode) {
+                    .incremental => break :blk try module.zig_cache_artifact_directory.join(
+                        arena,
+                        &[_][]const u8{obj_basename},
+                    ),
+                    .whole => break :blk try fs.path.join(arena, &.{
+                        fs.path.dirname(full_out_path_z).?, obj_basename,
+                    }),
+                }
             }
             if (base.options.object_format == .macho) {
                 try base.cast(MachO).?.flushObject(comp);
             } else {
                 try base.flushModule(comp);
             }
-            const obj_basename = base.intermediary_basename.?;
-            const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
-            break :blk full_obj_path;
+            break :blk try fs.path.join(arena, &.{
+                fs.path.dirname(full_out_path_z).?, base.intermediary_basename.?,
+            });
         } else null;
 
+        log.debug("module_obj_path={s}", .{if (module_obj_path) |s| s else "(null)"});
+
         const compiler_rt_path: ?[]const u8 = if (base.options.include_compiler_rt)
             comp.compiler_rt_obj.?.full_object_path
         else
@@ -742,9 +782,6 @@ pub const File = struct {
             object_files.appendAssumeCapacity(try arena.dupeZ(u8, p));
         }
 
-        const full_out_path = try directory.join(arena, &[_][]const u8{base.options.emit.?.sub_path});
-        const full_out_path_z = try arena.dupeZ(u8, full_out_path);
-
         if (base.options.verbose_link) {
             std.debug.print("ar rcs {s}", .{full_out_path_z});
             for (object_files.items) |arg| {
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index e6788120a050..ec06c28b448a 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -880,6 +880,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
     const arena = arena_allocator.allocator();
 
     const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
 
     // If there is no Zig code to compile, then we should skip flushing the output file because it
     // will not be part of the linker line anyway.
@@ -891,15 +892,22 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
                 .target = self.base.options.target,
                 .output_mode = .Obj,
             });
-            const o_directory = module.zig_cache_artifact_directory;
-            const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
-            break :blk full_obj_path;
+            switch (self.base.options.cache_mode) {
+                .incremental => break :blk try module.zig_cache_artifact_directory.join(
+                    arena,
+                    &[_][]const u8{obj_basename},
+                ),
+                .whole => break :blk try fs.path.join(arena, &.{
+                    fs.path.dirname(full_out_path).?, obj_basename,
+                }),
+            }
         }
 
         try self.flushModule(comp);
-        const obj_basename = self.base.intermediary_basename.?;
-        const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
-        break :blk full_obj_path;
+
+        break :blk try fs.path.join(arena, &.{
+            fs.path.dirname(full_out_path).?, self.base.intermediary_basename.?,
+        });
     } else null;
 
     const is_lib = self.base.options.output_mode == .Lib;
@@ -978,7 +986,6 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         };
     }
 
-    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
     if (self.base.options.output_mode == .Obj) {
         // LLD's COFF driver does not support the equivalent of `-r` so we do a simple file copy
         // here. TODO: think carefully about how we can avoid this redundant operation when doing
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 5f3771a3cb5b..105b012fbfb4 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -297,6 +297,7 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {
         else => return error.UnsupportedELFArchitecture,
     };
     const self = try gpa.create(Elf);
+    errdefer gpa.destroy(self);
     self.* = .{
         .base = .{
             .tag = .elf,
@@ -306,6 +307,9 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {
         },
         .ptr_width = ptr_width,
     };
+    // TODO get rid of the sub_path parameter to LlvmObject.create
+    // and create the llvm_object here. Also openPath needs to
+    // not override this field or there will be a memory leak.
     return self;
 }
 
@@ -1298,6 +1302,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     const arena = arena_allocator.allocator();
 
     const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
 
     // If there is no Zig code to compile, then we should skip flushing the output file because it
     // will not be part of the linker line anyway.
@@ -1309,15 +1314,22 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
                 .target = self.base.options.target,
                 .output_mode = .Obj,
             });
-            const o_directory = module.zig_cache_artifact_directory;
-            const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
-            break :blk full_obj_path;
+            switch (self.base.options.cache_mode) {
+                .incremental => break :blk try module.zig_cache_artifact_directory.join(
+                    arena,
+                    &[_][]const u8{obj_basename},
+                ),
+                .whole => break :blk try fs.path.join(arena, &.{
+                    fs.path.dirname(full_out_path).?, obj_basename,
+                }),
+            }
         }
 
         try self.flushModule(comp);
-        const obj_basename = self.base.intermediary_basename.?;
-        const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
-        break :blk full_obj_path;
+
+        break :blk try fs.path.join(arena, &.{
+            fs.path.dirname(full_out_path).?, self.base.intermediary_basename.?,
+        });
     } else null;
 
     const is_obj = self.base.options.output_mode == .Obj;
@@ -1434,8 +1446,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         };
     }
 
-    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
-
     // Due to a deficiency in LLD, we need to special-case BPF to a simple file copy when generating
     // relocatables. Normally, we would expect `lld -r` to work. However, because LLD wants to resolve
     // BPF relocations which it shouldn't, it fails before even generating the relocatable.
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index f970ffb5f206..ed720080af02 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -423,6 +423,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
     const arena = arena_allocator.allocator();
 
     const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
 
     // If there is no Zig code to compile, then we should skip flushing the output file because it
     // will not be part of the linker line anyway.
@@ -433,15 +434,24 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
                 .target = self.base.options.target,
                 .output_mode = .Obj,
             });
-            const o_directory = module.zig_cache_artifact_directory;
-            const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
-            break :blk full_obj_path;
+            switch (self.base.options.cache_mode) {
+                .incremental => break :blk try module.zig_cache_artifact_directory.join(
+                    arena,
+                    &[_][]const u8{obj_basename},
+                ),
+                .whole => break :blk try fs.path.join(arena, &.{
+                    fs.path.dirname(full_out_path).?, obj_basename,
+                }),
+            }
         }
 
         const obj_basename = self.base.intermediary_basename orelse break :blk null;
+
         try self.flushObject(comp);
-        const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
-        break :blk full_obj_path;
+
+        break :blk try fs.path.join(arena, &.{
+            fs.path.dirname(full_out_path).?, obj_basename,
+        });
     } else null;
 
     const is_lib = self.base.options.output_mode == .Lib;
@@ -534,7 +544,6 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
             else => |e| return e,
         };
     }
-    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
 
     if (self.base.options.output_mode == .Obj) {
         // LLD's MachO driver does not support the equivalent of `-r` so we do a simple file copy
@@ -1269,7 +1278,7 @@ fn parseInputFiles(self: *MachO, files: []const []const u8, syslibroot: ?[]const
     for (files) |file_name| {
         const full_path = full_path: {
             var buffer: [fs.MAX_PATH_BYTES]u8 = undefined;
-            const path = try std.fs.realpath(file_name, &buffer);
+            const path = try fs.realpath(file_name, &buffer);
             break :full_path try self.base.allocator.dupe(u8, path);
         };
         defer self.base.allocator.free(full_path);
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
index 5f8f48d1b19a..220ab2f53c68 100644
--- a/src/link/Wasm.zig
+++ b/src/link/Wasm.zig
@@ -1050,6 +1050,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
     const arena = arena_allocator.allocator();
 
     const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
 
     // If there is no Zig code to compile, then we should skip flushing the output file because it
     // will not be part of the linker line anyway.
@@ -1061,15 +1062,22 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
                 .target = self.base.options.target,
                 .output_mode = .Obj,
             });
-            const o_directory = module.zig_cache_artifact_directory;
-            const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
-            break :blk full_obj_path;
+            switch (self.base.options.cache_mode) {
+                .incremental => break :blk try module.zig_cache_artifact_directory.join(
+                    arena,
+                    &[_][]const u8{obj_basename},
+                ),
+                .whole => break :blk try fs.path.join(arena, &.{
+                    fs.path.dirname(full_out_path).?, obj_basename,
+                }),
+            }
         }
 
         try self.flushModule(comp);
-        const obj_basename = self.base.intermediary_basename.?;
-        const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
-        break :blk full_obj_path;
+
+        break :blk try fs.path.join(arena, &.{
+            fs.path.dirname(full_out_path).?, self.base.intermediary_basename.?,
+        });
     } else null;
 
     const is_obj = self.base.options.output_mode == .Obj;
@@ -1143,8 +1151,6 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
         };
     }
 
-    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
-
     if (self.base.options.output_mode == .Obj) {
         // LLD's WASM driver does not support the equivalent of `-r` so we do a simple file copy
         // here. TODO: think carefully about how we can avoid this redundant operation when doing