stdenv: Nix-driven bootstrap of gcc

 #### Summary

By default, when you type `make`, GCC will compile itself three
times.  This PR inhibits that behavior by configuring GCC with
`--disable-bootstrap`, and reimplements the triple-rebuild using
Nix rather than `make`/`sh`.

 #### Immediate Benefits

- Allow `gcc11` and `gcc12` on `aarch64` (without needing new
  `bootstrapFiles`)
- Faster stdenv rebuilds: the third compilation of gcc
  (i.e. stageCompare) is no longer a `drvInput` of the final stdenv.
  This allows Nix to build stageCompare in parallel with the rest of
  nixpkgs instead of in series.
- No more copying `libgcc_s` out of the bootstrap-files or other
  derivations
- No more Frankenstein compiler: the final gcc and the libraries it
  links against (mpfr, mpc, isl, glibc) are all built by the same
  compiler (xgcc) instead of a mixture of the bootstrapFiles'
  compiler and xgcc.
- No more [static lib{mpfr,mpc,gmp,isl}.a hack]
- Many other small `stdenv` hacks eliminated
- `gcc` and `clang` share the same codepath for more of `cc-wrapper`.

 #### Future Benefits

- This should allow using a [foreign] `bootstrap-files` so long as
  `hostPlatform.canExecute bootstrapFiles`.
- This should allow each of the libraries that ship with `gcc`
  (lib{backtrace, atomic, cc1, decnumber, ffi, gomp, iberty,
  offloadatomic, quadmath, sanitizer, ssp, stdc++-v3, vtv}) to be
  built in separate (one-liner) derivations which `inherit src;`
  from `gcc`, much like NixOS#132343

 #### Incorporates

- NixOS#210004
- NixOS#36948 (unreverted)
- NixOS#210325
- NixOS#210118
- NixOS#210132
- NixOS#210109
- NixOS#213909
- NixOS#216136
- NixOS#216237
- NixOS#210019
- NixOS#216232
- NixOS#216016
- NixOS#217977
- NixOS#217995

 #### Closes

- Closes NixOS#108305
- Closes NixOS#108111
- Closes NixOS#201254
- Closes NixOS#208412

 #### Credits

This project was made possible by three important insights, none of
which were mine:

1. @Ericson2314 was the first to advocate for this change, and
   probably the first to appreciate its advantages.  Nix-driven
   (external) bootstrap is "cross by default".

2. @trofi has figured out a lot about how to get gcc to not mix up
   the copy of `libstdc++` that it depends on with the copy that it
   builds, by moving the `bootstrapFiles`' `libstdc++` into a
   [versioned directory].  This allows a Nix-driven bootstrap of gcc
   without the final gcc would still having references to the
   `bootstrapFiles`.

3. Using the undocumented variable [`user-defined-trusted-dirs`]
   when building glibc.  When glibc `dlopen()`s `libgcc_s.so`, it
   uses a completely different and totally special set of rules for
   finding `libgcc_s.so`.  This trick is the only way we can put
   `libgcc_s.so` in its own separate outpath without creating
   circular dependencies or dependencies on the bootstrapFiles.  I
   would never have guessed to use this (or that it existed!) if it
   were not for a [comment in guix] which @Mic92 [mentioned].

My own role in this PR was basically: being available to go on a
coding binge at an opportune moment, so we wouldn't waste a
[crisis].

[aarch64-compare-ofborg]: https://github.com/NixOS/nixpkgs/pull/209870/checks?check_run_id=10662822938
[amd64-compare-ofborg]: https://github.com/NixOS/nixpkgs/pull/209870/checks?check_run_id=10662825857
[nonexistent sysroot]: NixOS#210004
[versioned directory]: NixOS#209054
[`user-defined-trusted-dirs`]: https://sourceware.org/legacy-ml/libc-help/2013-11/msg00026.html
[comment in guix]: https://github.com/guix-mirror/guix/blob/5e4ec8218142eee8e6e148e787381a5ef891c5b1/gnu/packages/gcc.scm#L253
[mentioned]: NixOS#210112 (comment)
[crisis]: NixOS#108305
[foreign]: NixOS#170857 (comment)
[static lib{mpfr,mpc,gmp,isl}.a hack]: https://github.com/NixOS/nixpkgs/blob/2f1948af9c984ebb82dfd618e67dc949755823e2/pkgs/stdenv/linux/default.nix#L380

pkgs/build-support/cc-wrapper/default.nix

@@ -17,9 +17,40 @@
 , isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null
 , buildPackages ? {}
 , libcxx ? null
+
+# Whether or not to add `-B` and `-L` to `nix-support/cc-{c,ld}flags`
+, useCcForLibs ?
+
+  # Always add these flags for Clang, because in order to compile (most
+  # software) it needs libraries that are shipped and compiled with gcc.
+  if isClang then true
+
+  # Never add these flags for a build!=host cross-compiler or a host!=target
+  # ("cross-built-native") compiler; currently nixpkgs has a special build
+  # path for these (`crossStageStatic`).  Hopefully at some point that build
+  # path will be merged with this one and this conditional will be removed.
+  else if (with stdenvNoCC; buildPlatform != hostPlatform || hostPlatform != targetPlatform) then false
+
+  # Never add these flags when wrapping the bootstrapFiles' compiler; it has a
+  # /usr/-like layout with everything smashed into a single outpath, so it has
+  # no trouble finding its own libraries.
+  else if (cc.passthru.isFromBootstrapFiles or false) then false
+
+  # Add these flags when wrapping `xgcc` (the first compiler that nixpkgs builds)
+  else if (cc.passthru.isXgcc or false) then true
+
+  # Add these flags when wrapping `stdenv.cc`
+  else if (cc.stdenv.cc.cc.passthru.isXgcc or false) then true
+
+  # Do not add these flags in any other situation.  This is `false` mainly to
+  # prevent these flags from being added when wrapping *old* versions of gcc
+  # (e.g. `gcc6Stdenv`), since they will cause the old gcc to get `-B` and
+  # `-L` flags pointing at the new gcc's libstdc++ headers.  Example failure:
+  # https://hydra.nixos.org/build/213125495
+  else false
+
+# the derivation at which the `-B` and `-L` flags added by `useCcForLibs` will point
 , gccForLibs ? if useCcForLibs then cc else null
-# same as `gccForLibs`, but generalized beyond clang
-, useCcForLibs ? isClang
 }:
 
 with lib;

pkgs/development/compilers/gcc/11/default.nix

@@ -49,7 +49,7 @@ with builtins;
 
 let majorVersion = "11";
     version = "${majorVersion}.3.0";
-    disableBootstrap = !(with stdenv; targetPlatform == hostPlatform && hostPlatform == buildPlatform);
+    disableBootstrap = true;
 
     inherit (stdenv) buildPlatform hostPlatform targetPlatform;
 
@@ -239,9 +239,8 @@ lib.pipe (stdenv.mkDerivation ({
   targetConfig = if targetPlatform != hostPlatform then targetPlatform.config else null;
 
   buildFlags =
-    let target =
-          lib.optionalString (profiledCompiler) "profiled" +
-          lib.optionalString (targetPlatform == hostPlatform && hostPlatform == buildPlatform && !disableBootstrap) "bootstrap";
+    let target = lib.optionalString (profiledCompiler) "profiled"
+      + lib.optionalString (targetPlatform == hostPlatform && hostPlatform == buildPlatform && !disableBootstrap) "bootstrap";
     in lib.optional (target != "") target;
 
   inherit (callFile ../common/strip-attributes.nix { })
@@ -300,4 +299,5 @@ lib.pipe (stdenv.mkDerivation ({
 // optionalAttrs (enableMultilib) { dontMoveLib64 = true; }
 ))
 [
+  (callPackage ../common/libgcc.nix   { inherit langC langCC langJit; })
 ]

pkgs/development/compilers/gcc/12/default.nix

@@ -54,7 +54,7 @@ with builtins;
 
 let majorVersion = "12";
     version = "${majorVersion}.2.0";
-    disableBootstrap = !(with stdenv; targetPlatform == hostPlatform && hostPlatform == buildPlatform);
+    disableBootstrap = true;
 
     inherit (stdenv) buildPlatform hostPlatform targetPlatform;
 
@@ -346,5 +346,6 @@ lib.pipe (stdenv.mkDerivation ({
 // optionalAttrs (enableMultilib) { dontMoveLib64 = true; }
 ))
 [
+  (callPackage ../common/libgcc.nix   { inherit langC langCC langJit; })
 ]
 

pkgs/development/compilers/gcc/common/libgcc.nix

@@ -0,0 +1,96 @@
+{ lib
+, stdenv
+, langC
+, langCC
+, langJit
+}:
+
+let
+  enableLibGccOutput = (with stdenv; targetPlatform == hostPlatform) && !langJit;
+in
+(pkg: pkg.overrideAttrs (previousAttrs: lib.optionalAttrs ((!langC) || langJit || enableLibGccOutput) {
+  outputs = previousAttrs.outputs ++ lib.optionals enableLibGccOutput [ "libgcc" ];
+  # This is a separate phase because gcc assembles its phase scripts
+  # in bash instead of nix (we should fix that).
+  preFixupPhases = (previousAttrs.preFixupPhases or []) ++ [ "preFixupLibGccPhase" ];
+  preFixupLibGccPhase =
+    # delete extra/unused builds of libgcc_s in non-langC builds
+    # (i.e. libgccjit, gnat, etc) to avoid potential confusion
+    lib.optionalString (!langC) ''
+      rm -f $out/lib/libgcc_s.so*
+    ''
+
+    # TODO(amjoseph): remove the `libgcc_s.so` symlinks below and replace them
+    # with a `-L${gccForLibs.libgcc}/lib` in cc-wrapper's
+    # `$out/nix-support/cc-flags`.  See also:
+    # - https://github.com/NixOS/nixpkgs/pull/209870#discussion_r1130614895
+    # - https://github.com/NixOS/nixpkgs/pull/209870#discussion_r1130635982
+    # - https://github.com/NixOS/nixpkgs/commit/404155c6acfa59456aebe6156b22fe385e7dec6f
+    #
+    # move `libgcc_s.so` into its own output, `$libgcc`
+    + lib.optionalString enableLibGccOutput (''
+      # move libgcc from lib to its own output (libgcc)
+      mkdir -p $libgcc/lib
+      mv    $lib/lib/libgcc_s.so      $libgcc/lib/
+      mv    $lib/lib/libgcc_s.so.1    $libgcc/lib/
+      ln -s $libgcc/lib/libgcc_s.so   $lib/lib/
+      ln -s $libgcc/lib/libgcc_s.so.1 $lib/lib/
+    ''
+    #
+    # Nixpkgs ordinarily turns dynamic linking into pseudo-static linking:
+    # libraries are still loaded dynamically, exactly which copy of each
+    # library is loaded is permanently fixed at compile time (via RUNPATH).
+    # For libgcc_s we must revert to the "impure dynamic linking" style found
+    # in imperative software distributions.  We must do this because
+    # `libgcc_s` calls `malloc()` and therefore has a `DT_NEEDED` for `libc`,
+    # which creates two problems:
+    #
+    #  1. A circular package dependency `glibc`<-`libgcc`<-`glibc`
+    #
+    #  2. According to the `-Wl,-rpath` flags added by Nixpkgs' `ld-wrapper`,
+    #     the two versions of `glibc` in the cycle above are actually
+    #     different packages.  The later one is compiled by this `gcc`, but
+    #     the earlier one was compiled by the compiler *that compiled* this
+    #     `gcc` (usually the bootstrapFiles).  In any event, the `glibc`
+    #     dynamic loader won't honor that specificity without namespaced
+    #     manual loads (`dlmopen()`).  Once a `libc` is present in the address
+    #     space of a process, that `libc` will be used to satisfy all
+    #     `DT_NEEDED`s for `libc`, regardless of `RUNPATH`s.
+    #
+    # So we wipe the RUNPATH using `patchelf --set-rpath ""`.  We can't use
+    # `patchelf --remove-rpath`, because at least as of patchelf 0.15.0 it
+    # will leave the old RUNPATH string in the file where the reference
+    # scanner can still find it:
+    #
+    #   https://github.com/NixOS/patchelf/issues/453
+    #
+    # Note: we might be using the bootstrapFiles' copy of patchelf, so we have
+    # to keep doing it this way until both the issue is fixed *and* all the
+    # bootstrapFiles are regenerated, on every platform.
+    #
+    # This patchelfing is *not* effectively equivalent to copying
+    # `libgcc_s` into `glibc`'s outpath.  There is one minor and one
+    # major difference:
+    #
+    # 1. (Minor): multiple builds of `glibc` (say, with different
+    #    overrides or parameters) will all reference a single store
+    #    path:
+    #
+    #      /nix/store/xxx...xxx-gcc-libgcc/lib/libgcc_s.so.1
+    #
+    #    This many-to-one referrer relationship will be visible in the store's
+    #    dependency graph, and will be available to `nix-store -q` queries.
+    #    Copying `libgcc_s` into each of its referrers would lose that
+    #    information.
+    #
+    # 2. (Major): by referencing `libgcc_s.so.1`, rather than copying it, we
+    #    are still able to run `nix-store -qd` on it to find out how it got
+    #    built!  Most importantly, we can see from that deriver which compiler
+    #    was used to build it (or if it is part of the unpacked
+    #    bootstrap-files).  Copying `libgcc_s.so.1` from one outpath to
+    #    another eliminates the ability to make these queries.
+    #
+    + ''
+      patchelf --set-rpath "" $libgcc/lib/libgcc_s.so.1
+    '');
+}))

pkgs/development/libraries/glibc/default.nix

@@ -72,33 +72,26 @@ in
         ]);
     };
 
-    # When building glibc from bootstrap-tools, we need libgcc_s at RPATH for
-    # any program we run, because the gcc will have been placed at a new
-    # store path than that determined when built (as a source for the
-    # bootstrap-tools tarball)
-    # Building from a proper gcc staying in the path where it was installed,
-    # libgcc_s will now be at {gcc}/lib, and gcc's libgcc will be found without
-    # any special hack.
-    # TODO: remove this hack. Things that rely on this hack today:
-    # - dejagnu: during linux bootstrap tcl SIGSEGVs
-    # - clang-wrapper in cross-compilation
-    # Last attempt: https://github.com/NixOS/nixpkgs/pull/36948
-    preInstall = lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) ''
-      if [ -f ${lib.getLib stdenv.cc.cc}/lib/libgcc_s.so.1 ]; then
-          mkdir -p $out/lib
-          cp ${lib.getLib stdenv.cc.cc}/lib/libgcc_s.so.1 $out/lib/libgcc_s.so.1
-          # the .so It used to be a symlink, but now it is a script
-          cp -a ${lib.getLib stdenv.cc.cc}/lib/libgcc_s.so $out/lib/libgcc_s.so
-          # wipe out reference to previous libc it was built against
-          chmod +w $out/lib/libgcc_s.so.1
-          # rely on default RUNPATHs of the binary and other libraries
-          # Do no force-pull wrong glibc.
-          patchelf --remove-rpath $out/lib/libgcc_s.so.1
-          # 'patchelf' does not remove the string itself. Wipe out
-          # string reference to avoid possible link to bootstrapTools
-          ${buildPackages.nukeReferences}/bin/nuke-refs $out/lib/libgcc_s.so.1
-      fi
-    '';
+    # glibc needs to `dlopen()` `libgcc_s.so` but does not link
+    # against it.  Furthermore, glibc doesn't use the ordinary
+    # `dlopen()` call to do this; instead it uses one which ignores
+    # most paths:
+    #
+    #   https://sourceware.org/legacy-ml/libc-help/2013-11/msg00026.html
+    #
+    # In order to get it to not ignore `libgcc_s.so`, we have to add its path to
+    # `user-defined-trusted-dirs`:
+    #
+    #   https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/Makefile;h=b509b3eada1fb77bf81e2a0ca5740b94ad185764#l1355
+    #
+    # Conveniently, this will also inform Nix of the fact that glibc depends on
+    # gcc.libgcc, since the path will be embedded in the resulting binary.
+    #
+    makeFlags =
+      (previousAttrs.makeFlags or [])
+      ++ lib.optionals (stdenv.cc.cc?libgcc) [
+        "user-defined-trusted-dirs=${stdenv.cc.cc.libgcc}/lib"
+      ];
 
     postInstall = previousAttrs.postInstall + (if stdenv.buildPlatform.canExecute stdenv.hostPlatform then ''
       echo SUPPORTED-LOCALES=C.UTF-8/UTF-8 > ../glibc-2*/localedata/SUPPORTED
@@ -181,6 +174,12 @@ in
 
     separateDebugInfo = true;
 
+    passthru =
+      (previousAttrs.passthru or {})
+      // lib.optionalAttrs (stdenv.cc.cc?libgcc) {
+        inherit (stdenv.cc.cc) libgcc;
+      };
+
   meta = (previousAttrs.meta or {}) // { description = "The GNU C Library"; };
 })