stdenv: external gcc bootstrap

#### Immediate Benefits - Allow `gcc11` on `aarch64` - No more copying `libgcc_s` out of the bootstrap-files or other derivations - No more [static `lib{mpfr,mpc,gmp,isl}.a` hack](https://github.com/NixOS/nixpkgs/blob/2f1948af9c984ebb82dfd618e67dc949755823e2/pkgs/stdenv/linux/default.nix#L380) - *Zero* additional `gcc` builds (stage1+stage2+stageCompare) - The `gcc` derivation builds `gcc` once instead of three times. - The libraries that are linked into the final `pkgs.gcc` (`mpfr`, `mpc`, `gmp`, `isl`, `glibc`) are built by `stdenv.__bootPkgs.gcc` rather than by the `bootstrapFiles`. No more Frankenstein compiler! - stageCompare runs **concurrently** with (not in series with) with `stdenv`'s dependees. - Many other `stdenv` hacks eliminated. - `gcc` and `clang` share the same codepath for more of `cc-wrapper`. - Makes the cross and native codepaths much more similar -- another step towards "cross by default". Note that *all* the changes in this PR are controlled by flags; no old codepaths need to be removed until/if we're completely certain that this is the right way to go. #### Future Benefits - This should allow using a [foreign] `bootstrap-files` so long as `hostPlatform.canExecute bootstrapFiles`. - There will be an "avalanche of simplification" when we set `enableGccExternalBootstrap=true` and run dead code elimination. It's really quite a huge amount of code that goes away. Native-gcc has its own special codepath in so many places, while cross-gcc and clang work the same way (and are much simpler). - This should allow each of the libraries that ship with `gcc` (`lib{backtrace,atomic,cc1,decnumber,ffi,gomp,iberty,offloadatomic,quadmath,sanitizer,ssp,stdc++-v3,vtv}`) to be built in separate (one-liner) derivations which `inherit src;` from `gcc`. - Building `libstdc++-v3` in a separate derivation will eliminate a lot of accidental-reference-to-the-`bootstrapFiles` landmines. #### Incorporates - NixOS#209054 - NixOS#210004 - NixOS#36948 (unreverted) - NixOS#210325 - NixOS#210118 - NixOS#210132 - NixOS#210109 #### Closes - Closes NixOS#208412 - Closes NixOS#108111 - Closes NixOS#108305 - Closes NixOS#201254 #### Build history - First successful builds (stage1/stage2): - powerpc64le-linux at 9c7e9ef - x86_64-linux at 9c7e9ef - aarch64-linux at 4d5bc7d - First successful comparisons (stageCompare): - at 81949cf - [aarch64-linux][aarch64-compare-ofborg] - [x86\_64-linux][amd64-compare-ofborg] #### Credits This project was made possible by three important insights, none of which were mine: 1. @Ericson2314 was the first to advocate for this change, and probably the first to appreciate its advantages. External bootstrap is "cross by default". 2. @trofi has figured out a lot about how to get gcc to not mix up the copy of `libstdc++` that it depends on with the copy that it builds. Now that gcc is written in C++, it depends on `libstdc++`, builds a copy of `libstdc++`, and builds auxiliary products (like `libplugin`) which depend on `libstdc++`. @trofi developed two important techniques for keeping this straight: the use of a [nonexistent sysroot] and moving the `bootstrapFiles`' `libstdc++` into a [versioned directory]. Without these two discoveries, external bootstrap would be impossible, because the final gcc would still have references to the `bootstrapFiles`. 3. Using the undocumented variable [`user-defined-trusted-dirs`] when building glibc. When glibc `dlopen()`s `libgcc_s.so`, it uses a completely different and totally special set of rules for finding `libgcc_s.so`. This trick is the only way we can put `libgcc_s.so` in its own separate outpath without creating circular dependencies or dependencies on the bootstrapFiles. I would never have guessed to use this (or that it existed!) if it were not for a [comment in guix] which @Mic92 [mentioned]. My own role in this PR was basically: being available to go on a coding binge at an opportune moment, so we wouldn't waste a [crisis]. [aarch64-compare-ofborg]: https://github.com/NixOS/nixpkgs/pull/209870/checks?check_run_id=10662822938 [amd64-compare-ofborg]: https://github.com/NixOS/nixpkgs/pull/209870/checks?check_run_id=10662825857 [nonexistent sysroot]: NixOS#210004 [versioned directory]: NixOS#209054 [`user-defined-trusted-dirs`]: https://sourceware.org/legacy-ml/libc-help/2013-11/msg00026.html [comment in guix]: https://github.com/guix-mirror/guix/blob/5e4ec8218142eee8e6e148e787381a5ef891c5b1/gnu/packages/gcc.scm#L253 [mentioned]: NixOS#210112 (comment) [crisis]: NixOS#108305 [foreign]: NixOS#170857 (comment)
wegank · Feb 1, 2023 · fc823fe · fc823fe
1 parent 6527aa6
commit fc823fe
Show file tree

Hide file tree

Showing 20 changed files with 518 additions and 110 deletions.
diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix
@@ -60,9 +60,16 @@ assert withPgtk -> withGTK3 && !withX && gtk3 != null;
 assert withXwidgets -> withGTK3 && webkitgtk != null;
 
 
-let emacs = (if withMacport then llvmPackages_6.stdenv else stdenv).mkDerivation (lib.optionalAttrs nativeComp {
+let
+  libGccJitLibraryPaths = [
+    "${lib.getLib libgccjit}/lib/gcc"
+    "${lib.getLib stdenv.cc.libc}/lib"
+  ] ++ lib.optionals (stdenv.cc?cc.libgcc) [
+    "${lib.getLib stdenv.cc.cc.libgcc}/lib"
+  ];
+  emacs = (if withMacport then llvmPackages_6.stdenv else stdenv).mkDerivation (lib.optionalAttrs nativeComp {
   NATIVE_FULL_AOT = "1";
-  LIBRARY_PATH = "${lib.getLib stdenv.cc.libc}/lib";
+  LIBRARY_PATH = lib.concatStringsSep ":" libGccJitLibraryPaths;
 } // {
   pname = pname + lib.optionalString ( !withX && !withNS && !withMacport && !withGTK2 && !withGTK3 ) "-nox";
   inherit version;
@@ -73,17 +80,15 @@ let emacs = (if withMacport then llvmPackages_6.stdenv else stdenv).mkDerivation
             then ./native-comp-driver-options-28.patch
             else ./native-comp-driver-options.patch;
       backendPath = (lib.concatStringsSep " "
-        (builtins.map (x: ''"-B${x}"'') [
+        (builtins.map (x: ''"-B${x}"'') ([
           # Paths necessary so the JIT compiler finds its libraries:
           "${lib.getLib libgccjit}/lib"
-          "${lib.getLib libgccjit}/lib/gcc"
-          "${lib.getLib stdenv.cc.libc}/lib"
-
+        ] ++ libGccJitLibraryPaths ++ [
           # Executable paths necessary for compilation (ld, as):
           "${lib.getBin stdenv.cc.cc}/bin"
           "${lib.getBin stdenv.cc.bintools}/bin"
           "${lib.getBin stdenv.cc.bintools.bintools}/bin"
-        ]));
+        ])));
     })
   ];
 

diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix
@@ -108,6 +108,7 @@ stdenv.mkDerivation {
   outputs = [ "out" ] ++ optionals propagateDoc ([ "man" ] ++ optional (bintools ? info) "info");
 
   passthru = {
+    inherit (bintools.passthru) isFromBootstrapFiles;
     inherit targetPrefix suffixSalt;
     inherit bintools libc nativeTools nativeLibc nativePrefix isGNU isLLVM;
 

diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix
@@ -9,7 +9,7 @@
 , lib
 , stdenvNoCC
 , cc ? null, libc ? null, bintools, coreutils ? null, shell ? stdenvNoCC.shell
-, gccForLibs ? null
+, gccForLibs ? if useCcForLibs then cc else null
 , zlib ? null
 , nativeTools, noLibc ? false, nativeLibc, nativePrefix ? ""
 , propagateDoc ? cc != null && cc ? man
@@ -18,6 +18,7 @@
 , isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null
 , buildPackages ? {}
 , libcxx ? null
+, useCcForLibs ? isClang || (cc.passthru.enableExternalBootstrap or false)
 }:
 
 with lib;
@@ -63,7 +64,7 @@ let
     then import ../expand-response-params { inherit (buildPackages) stdenv; }
     else "";
 
-  useGccForLibs = isClang
+  useGccForLibs = useCcForLibs
     && libcxx == null
     && !stdenv.targetPlatform.isDarwin
     && !(stdenv.targetPlatform.useLLVM or false)
@@ -155,6 +156,7 @@ stdenv.mkDerivation {
     inherit expand-response-params;
 
     inherit nixSupport;
+    inherit (cc.passthru) isFromBootstrapFiles;
   };
 
   dontBuild = true;

diff --git a/pkgs/build-support/emacs/wrapper.nix b/pkgs/build-support/emacs/wrapper.nix
@@ -65,10 +65,12 @@ runCommand
     # Store all paths we want to add to emacs here, so that we only need to add
     # one path to the load lists
     deps = runCommand "emacs-packages-deps"
-      {
+      ({
         inherit explicitRequires lndir emacs;
         nativeBuildInputs = lib.optional nativeComp gcc;
-      }
+      } // lib.optionalAttrs nativeComp {
+        inherit (emacs) LIBRARY_PATH;
+      })
       ''
         findInputsOld() {
           local pkg="$1"; shift

diff --git a/pkgs/build-support/setup-hooks/reproducible-builds.sh b/pkgs/build-support/setup-hooks/reproducible-builds.sh
@@ -3,7 +3,8 @@
 # derivation and not easily collide with other builds.
 # We also truncate the hash so that it cannot cause reference cycles.
 NIX_CFLAGS_COMPILE="${NIX_CFLAGS_COMPILE:-} -frandom-seed=$(
-    outbase="${out##*/}"
+    randSeed=${NIX_OUTPATH_USED_AS_RANDOM_SEED:-$out}
+    outbase="${randSeed##*/}"
     randomseed="${outbase:0:10}"
     echo $randomseed
 )"

diff --git a/pkgs/development/compilers/gcc/11/default.nix b/pkgs/development/compilers/gcc/11/default.nix
@@ -19,6 +19,7 @@
 , gnatboot ? null
 , enableMultilib ? false
 , enablePlugin ? stdenv.hostPlatform == stdenv.buildPlatform # Whether to support user-supplied plug-ins
+, enableGdbPlugin ? enablePlugin
 , name ? "gcc"
 , libcCross ? null
 , threadsCross ? null # for MinGW
@@ -27,8 +28,17 @@
 , cloog # unused; just for compat with gcc4, as we override the parameter on some places
 , buildPackages
 , libxcrypt
+, nukeReferences
+, enableExternalBootstrap
+  ? (with stdenv; targetPlatform == hostPlatform && hostPlatform == buildPlatform)
+    && stdenv.enableGccExternalBootstrapForStdenv or false
+, enableLibGccOutput
+  ? (with stdenv; targetPlatform == hostPlatform)
 }:
 
+# libgccjit.so must be built separately
+assert langJit -> !enableLibGccOutput;
+
 # Make sure we get GNU sed.
 assert stdenv.buildPlatform.isDarwin -> gnused != null;
 
@@ -89,9 +99,10 @@ let majorVersion = "11";
     stageNameAddon = if crossStageStatic then "stage-static" else "stage-final";
     crossNameAddon = optionalString (targetPlatform != hostPlatform) "${targetPlatform.config}-${stageNameAddon}-";
 
+    enableChecksum = enableExternalBootstrap && langC && langCC;
 in
 
-stdenv.mkDerivation ({
+lib.pipe (stdenv.mkDerivation ({
   pname = "${crossNameAddon}${name}";
   inherit version;
 
@@ -104,7 +115,10 @@ stdenv.mkDerivation ({
 
   inherit patches;
 
-  outputs = [ "out" "man" "info" ] ++ lib.optional (!langJit) "lib";
+  outputs = [ "out" "man" "info" ]
+  ++ lib.optional (!langJit) "lib"
+  ++ lib.optionals enableLibGccOutput [ "libgcc" ]
+  ++ lib.optionals enableChecksum [ "checksum" ];
   setOutputFlags = false;
   NIX_NO_SELF_RPATH = true;
 
@@ -165,7 +179,7 @@ stdenv.mkDerivation ({
     libcCross crossMingw;
 
   depsBuildBuild = [ buildPackages.stdenv.cc ];
-  nativeBuildInputs = [ texinfo which gettext ]
+  nativeBuildInputs = [ texinfo which gettext nukeReferences ]
     ++ (optional (perl != null) perl)
     ++ (optional langAda gnatboot)
     # The builder relies on GNU sed (for instance, Darwin's `sed' fails with
@@ -207,6 +221,7 @@ stdenv.mkDerivation ({
   configurePlatforms = [ "build" "host" "target" ];
 
   configureFlags = import ../common/configure-flags.nix {
+    enableInternalBootstrap = targetPlatform == hostPlatform && hostPlatform == buildPlatform && !enableExternalBootstrap;
     inherit
       lib
       stdenv
@@ -219,6 +234,7 @@ stdenv.mkDerivation ({
       enableLTO
       enableMultilib
       enablePlugin
+      enableGdbPlugin
       enableShared
 
       langC
@@ -238,7 +254,10 @@ stdenv.mkDerivation ({
 
   buildFlags = optional
     (targetPlatform == hostPlatform && hostPlatform == buildPlatform)
-    (if profiledCompiler then "profiledbootstrap" else "bootstrap");
+    (let
+      bootstrap = lib.optionalString (!enableExternalBootstrap) "bootstrap";
+      profiled  = lib.optionalString profiledCompiler "profiled";
+    in "${profiled}${bootstrap}");
 
   inherit
     (import ../common/strip-attributes.nix { inherit lib stdenv langJit; })
@@ -275,6 +294,8 @@ stdenv.mkDerivation ({
   passthru = {
     inherit langC langCC langObjC langObjCpp langAda langFortran langGo langD version;
     isGNU = true;
+    isFromBootstrapFiles = false;
+    inherit enableExternalBootstrap;
   };
 
   enableParallelBuilding = true;
@@ -305,5 +326,132 @@ stdenv.mkDerivation ({
   installTargets = "install-gcc install-target-libgcc";
 }
 
-// optionalAttrs (enableMultilib) { dontMoveLib64 = true; }
+// optionalAttrs (enableMultilib) { dontMoveLib64 = true; })
 )
+  [
+    (pkg: pkg.overrideAttrs (previousAttrs: optionalAttrs ((!langC) || langJit || enableLibGccOutput) {
+      # This is a separate phase because gcc assembles its phase scripts
+      # in bash instead of nix (we should fix that).
+      preFixupPhases = (previousAttrs.preFixupPhases or []) ++ [ "postPostInstallPhase" ];
+      postPostInstallPhase = lib.optionalString langJit ''
+        # this is to keep clang happy
+        mv $out/lib/gcc/${targetPlatform.config}/${version}/* $out/lib/gcc/
+        rmdir $out/lib/gcc/${targetPlatform.config}/${version}
+        rmdir $out/lib/gcc/${targetPlatform.config}
+      '' + lib.optionalString enableLibGccOutput ''
+        # eliminate false lib->out references
+        find $lib/lib/ -name \*.so\* -exec patchelf --shrink-rpath {} \; || true
+      '' + lib.optionalString (!langC) ''
+        # delete extra/unused builds of libgcc_s to avoid potential confusion:
+        rm -f $out/lib/libgcc_s.so*
+      '' + lib.optionalString enableLibGccOutput (''
+        # move libgcc from lib to its own output (libgcc)
+        mkdir -p $libgcc/lib
+        mv    $lib/lib/libgcc_s.so       $libgcc/lib/
+        mv    $lib/lib/libgcc_s.so.1     $libgcc/lib/
+        ln -s $libgcc/lib/libgcc_s.so   $lib/lib/
+        ln -s $libgcc/lib/libgcc_s.so.1 $lib/lib/
+      ''
+      #
+      # Nixpkgs ordinarily turns dynamic linking into pseudo-static linking:
+      # libraries are still loaded dynamically, exactly which copy of each
+      # library is loaded is permanently fixed at compile time (via RUNPATH).
+      # For libgcc_s we must revert to the "impure dynamic linking" style found
+      # in imperative software distributions for `libgcc_s`.  This is because
+      # `libgcc_s` calls `malloc()` and therefore has a `DT_NEEDED` for `libc`,
+      # which creates two problems:
+      #
+      #  1. A circular package dependency `glibc`<-`libgcc`<-`glibc`
+      #
+      #  2. According to the `-Wl,-rpath` flags added by Nixpkgs' `ld-wrapper`,
+      #     the two versions of `glibc` in the cycle above are actually
+      #     different packages.  The later one is compiled by this `gcc`, but
+      #     the earlier one was compiled by the compiler *that compiled* this
+      #     `gcc` (usually the bootstrapFiles).  In any event, the `glibc`
+      #     dynamic loader won't honor that specificity without namespaced
+      #     manual loads (`dlmopen()`).  Once a `libc` is present in the address
+      #     space of a process, that `libc` will be used to satisfy all
+      #     `DT_NEEDED`s for `libc`, regardless of `RUNPATH`s.
+      #
+      # So we wipe the RUNPATH:
+      #
+      + ''
+        patchelf --set-rpath "" $libgcc/lib/libgcc_s.so.1
+      '');
+      #
+      # Note: `patchelf --remove-rpath` up through (and possibly after) version
+      # 0.15.0 will leave the old RUNPATH string in the file where the reference
+      # scanner can still find it:
+      #
+      #   https://github.com/NixOS/patchelf/issues/453
+      #
+      # ...so we use `--set-rpath ""` instead.  We'll have to keep doing it this
+      # way even after that issue is fixed because we might be using the
+      # bootstrapFiles' copy of patchelf.
+      #
+      # That the patchelfing above is *not* effectively equivalent to copying
+      # `libgcc_s` into `glibc`'s outpath.  There is one minor and one major
+      # difference:
+      #
+      # 1. (Minor): multiple builds of `glibc` (say, with different
+      #    overrides or parameters) will all reference a single store
+      #    path:
+      #
+      #      /nix/store/xxx...xxx-gcc-libgcc/lib/libgcc_s.so.1
+      #
+      #    This many-to-one referrer relationship will be visible in the store's
+      #    dependency graph, and will be available to `nix-store -q` queries.
+      #    Copying `libgcc_s` into each of its referrers would lose that
+      #    information.
+      #
+      # 2. (Major): by referencing `libgcc_s.so.1`, rather than copying it, we
+      #    are still able to run `nix-store -qd` on it to find out how it got
+      #    built!  Most importantly, we can see from that deriver which compiler
+      #    was used to build it (or if it is part of the unpacked
+      #    bootstrap-files).  Copying `libgcc_s.so.1` from one outpath to
+      #    another eliminates the ability to make these queries.
+      #
+    }))
+
+    (pkg: pkg.overrideAttrs (previousAttrs: lib.optionalAttrs enableChecksum {
+      # This is a separate phase because gcc assembles its phase scripts
+      # in bash instead of nix (we should fix that).
+      preFixupPhases = (previousAttrs.preFixupPhases or []) ++ [ "postInstallSaveChecksumPhase" ];
+      #
+      # gcc uses an auxiliary utility `genchecksum` to md5-hash (most of) its
+      # `.o` and `.a` files prior to linking (in case the linker is
+      # nondeterministic).  Since we want to compare across gccs built from two
+      # separate derivations, we wrap `genchecksum` with a `nuke-references`
+      # call.  We also stash copies of the inputs to `genchecksum` in
+      # `$checksum/inputs/` -- this is extremely helpful for debugging since
+      # it's hard to get Nix to not delete the $NIX_BUILD_TOP of a successful
+      # build.
+      #
+      postInstallSaveChecksumPhase = ''
+        mv gcc/build/genchecksum gcc/build/.genchecksum-wrapped
+        cat > gcc/build/genchecksum <<\EOF
+        #!/bin/sh
+        ${nukeReferences}/bin/nuke-refs $@
+        for INPUT in "$@"; do install -Dt $INPUT $checksum/inputs/; done
+        exec build/.genchecksum-wrapped $@
+        EOF
+        chmod +x gcc/build/genchecksum
+        rm gcc/*-checksum.*
+        make -C gcc cc1-checksum.o cc1plus-checksum.o
+        install -Dt $checksum/checksums/ gcc/cc*-checksum.o
+      '';
+    }))
+    # the plugins reference $out, so we can't put them in $lib
+    (pkg: pkg.overrideAttrs (previousAttrs: lib.optionalAttrs (enableExternalBootstrap && enablePlugin) {
+      # This is a separate phase because gcc assembles its phase scripts
+      # in bash instead of nix (we should fix that).
+      preFixupPhases = (previousAttrs.preFixupPhases or []) ++ [ "postInstallPluginPhase" ];
+      postInstallPluginPhase =
+        let path = "lib/gcc/${targetPlatform.config}/${previousAttrs.version}/"; in ''
+          if [ -e "$lib/${path}/plugin" ]; then
+            mkdir -p $out/${path}
+            mv $lib/${path}/plugin $out/${path}/plugin
+          fi
+        '';
+    }))
+  ]
diff --git a/pkgs/development/compilers/gcc/9/default.nix b/pkgs/development/compilers/gcc/9/default.nix
@@ -267,6 +267,7 @@ stdenv.mkDerivation ({
   passthru = {
     inherit langC langCC langObjC langObjCpp langAda langFortran langGo langD version;
     isGNU = true;
+    isFromBootstrapFiles = false;
   };
 
   enableParallelBuilding = true;

diff --git a/pkgs/development/compilers/gcc/common/configure-flags.nix b/pkgs/development/compilers/gcc/common/configure-flags.nix
@@ -11,6 +11,7 @@
 , enableLTO
 , enableMultilib
 , enablePlugin
+, enableGdbPlugin ? enablePlugin  # "Libcc1 is the GCC cc1 plugin for the GDB debugger"
 , enableShared
 
 , langC
@@ -23,8 +24,12 @@
 , langObjC
 , langObjCpp
 , langJit
+, enableInternalBootstrap
+  ? (with stdenv; targetPlatform == hostPlatform && hostPlatform == buildPlatform)
+    && throw "please specify enableInternalBootstrap explicitly for native builds"
 }:
 
+assert enableGdbPlugin -> enablePlugin;
 assert cloog != null -> lib.versionOlder version "5";
 assert langJava -> lib.versionOlder version "7";
 
@@ -172,9 +177,8 @@ let
       then ["--enable-multilib" "--disable-libquadmath"]
       else ["--disable-multilib"])
     ++ lib.optional (!enableShared) "--disable-shared"
-    ++ [
-      (lib.enableFeature enablePlugin "plugin")
-    ]
+    ++ [ (lib.enableFeature enablePlugin "plugin") ]
+    ++ [ (lib.enableFeature enableGdbPlugin "libcc1") ]
 
     # Support -m32 on powerpc64le/be
     ++ lib.optional (targetPlatform.system == "powerpc64le-linux")
@@ -215,7 +219,7 @@ let
     # TODO: aarch64-darwin has clang stdenv and its arch and cpu flag values are incompatible with gcc
     ++ lib.optionals (!(stdenv.isDarwin && stdenv.isAarch64)) (import ../common/platform-flags.nix { inherit (stdenv)  targetPlatform; inherit lib; })
     ++ lib.optionals (targetPlatform != hostPlatform) crossConfigureFlags
-    ++ lib.optional (targetPlatform != hostPlatform) "--disable-bootstrap"
+    ++ lib.optional (!enableInternalBootstrap) "--disable-bootstrap"
 
     # Platform-specific flags
     ++ lib.optional (targetPlatform == hostPlatform && targetPlatform.isx86_32) "--with-arch=${stdenv.hostPlatform.parsed.cpu.name}"
@@ -240,6 +244,9 @@ let
     ++ lib.optionals (langD) [
       "--with-target-system-zlib=yes"
     ]
+    # stdenv's setOutputFlags puts libexec in $lib; we need it in
+    # $out to prevent circular references between $lib and $out
+    ++ [ "--libexecdir=${builtins.placeholder "out"}/libexec" ]
   ;
 
 in configureFlags