Skip to content

Commit

Permalink
stdenv/linux/default.nix: add gcc rebuild during bootstrap
Browse files Browse the repository at this point in the history
Before the change our rebuild chain was the followin:

- `bootstrapTools` install
- `binutils` built by `bootstrapTools`
- `glibc` built by `bootstrapTools`
- `gcc` built by `bootstrapTools`

As a result `glibc` contained code generated by `bootstrapTools`'s
`gcc`. And (what worse) copied `libgcc_s.so.1` as is from
`bootstrapTools`'s `gcc`.

Such `libgcc_s.so.1` was not compatible with `nixpkgs` `gcc` at
least on `aarch64` where newer `libgcc` is expected to have new symbols.
As a result linking failed as:

    ld: /build/test.o: in function `foo(int)':
    test.cpp:(.text+0x2c): undefined reference to `__aarch64_ldadd4_acq_rel'
    collect2: error: ld returned 1 exit status

The change rejigs rebuild sequence as:

- `bootstrapTools` install
- `binutils` built by `bootstrapTools`
- `gcc` built by `bootstrapTools`
- `glibc` built by `gcc`
- `gcc` built by `gcc`

As a result `glibc` gets built by fresher `gcc` and embeds a copy
of `libgcc_s.so.1`.

Co-authored-by: Rick van Schijndel <Mindavi@users.noreply.github.com>
  • Loading branch information
trofi and Mindavi committed Jan 14, 2023
1 parent 6728277 commit 8ee2b7c
Showing 1 changed file with 67 additions and 42 deletions.
109 changes: 67 additions & 42 deletions pkgs/stdenv/linux/default.nix
Expand Up @@ -9,14 +9,11 @@
# is used to build all other packages (including the bootstrapFiles).
#
# Goals of the bootstrap process:
# 1. final stdenv must not reference any of the bootstrap files.
# 2. final stdenv must not contain any of the bootstrap files
# (the only current violation is libgcc_s.so in glibc).
# 1. final stdenv must not reference any of the bootstrap packages.
# 2. final stdenv must not contain any of the bootstrap files.
# 3. final stdenv must not contain any of the files directly
# generated by the bootstrap code generators (assembler, linker,
# compiler). The only current violations are: libgcc_s.so in glibc,
# the lib{mpfr,mpc,gmp,isl} which are statically linked
# into the final gcc).
# compiler).
#
# These goals ensure that final packages and final stdenv are built
# exclusively using nixpkgs package definitions and don't depend
Expand Down Expand Up @@ -109,12 +106,13 @@ let
'';


# The bootstrap process proceeds in several steps.


# Create a standard environment by downloading pre-built binaries of
# coreutils, GCC, etc.

# The bootstrap process proceeds in several steps. Our high-level plan
# is the following:
#
# - build binutils with bootstrapTools
# - build gcc with bootstrapTools
# - build glibc with bootstrapTools + gcc from nixpkgs (embed libgcc_s.so.1)
# - build gcc with nixpkgs toolchain

# Download and unpack the bootstrap tools (coreutils, GCC, Glibc, ...).
bootstrapTools = import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) {
Expand Down Expand Up @@ -284,16 +282,56 @@ in
};
})


# 2nd stdenv that contains our own rebuilt binutils and is used for
# compiling our own Glibc.
# compiling our own gcc.
#
# resulting stage2 stdenv:
# - coreutils, glibc, gcc: from bootstrapFiles
# - binutils: from nixpkgs, built by bootstrapFiles toolchain
# - coreutils, glibc: from bootstrapFiles
# - binutils, gcc: from nixpkgs, built by bootstrapFiles toolchain
(prevStage: stageFun prevStage {
name = "bootstrap-stage2";

overrides = self: super: {
inherit (prevStage)
ccWrapperStdenv
coreutils gnugrep
perl gnum4 bison;

${localSystem.libc} = getLibc prevStage;

gcc-unwrapped =
let makeStaticLibrariesAndMark = pkg:
lib.makeOverridable (pkg.override { stdenv = self.makeStaticLibraries self.stdenv; })
.overrideAttrs (a: { pname = "${a.pname}-stage2"; });
in super.gcc-unwrapped.override {
# Link GCC statically against GMP etc. This makes sense because
# these builds of the libraries are only used by GCC, so it
# reduces the size of the stdenv closure.
gmp = makeStaticLibrariesAndMark super.gmp;
mpfr = makeStaticLibrariesAndMark super.mpfr;
libmpc = makeStaticLibrariesAndMark super.libmpc;
isl = makeStaticLibrariesAndMark super.isl_0_20;
# Use a deterministically built compiler
# see https://github.com/NixOS/nixpkgs/issues/108475 for context
reproducibleBuild = true;
profiledCompiler = false;
};
};

# `libtool` comes with obsolete config.sub/config.guess that don't recognize Risc-V.
extraNativeBuildInputs =
lib.optional (localSystem.isRiscV) prevStage.updateAutotoolsGnuConfigScriptsHook;
})

# 3rd stdenv that contains our own rebuilt binutils and gcc and is
# used for compiling our own Glibc.
#
# resulting stage3 stdenv:
# - coreutils: from bootstrapFiles
# - glibc, binutils, gcc: from nixpkgs, built by bootstrapFiles toolchain
(prevStage: stageFun prevStage {
name = "bootstrap-stage3";

overrides = self: super: {
inherit (prevStage)
ccWrapperStdenv
Expand Down Expand Up @@ -357,15 +395,16 @@ in
})


# Construct a third stdenv identical to the 2nd, except that this
# Construct a fourth stdenv identical to the 3rd, except that this
# one uses the rebuilt Glibc from stage2. It still uses the recent
# binutils and rest of the bootstrap tools, including GCC.
#
# resulting stage3 stdenv:
# resulting stage4 stdenv:
# - coreutils, gcc: from bootstrapFiles
# - glibc, binutils: from nixpkgs, built by bootstrapFiles toolchain
# - binutils, gcc: from nixpkgs, built by bootstrapFiles toolchain
# - glibc: from nixpkgs, built by nixpkgs toolchain
(prevStage: stageFun prevStage {
name = "bootstrap-stage3";
name = "bootstrap-stage4";

overrides = self: super: rec {
inherit (prevStage)
Expand All @@ -376,7 +415,7 @@ in
gcc-unwrapped =
let makeStaticLibrariesAndMark = pkg:
lib.makeOverridable (pkg.override { stdenv = self.makeStaticLibraries self.stdenv; })
.overrideAttrs (a: { pname = "${a.pname}-stage3"; });
.overrideAttrs (a: { pname = "${a.pname}-stage4"; });
in super.gcc-unwrapped.override {
# Link GCC statically against GMP etc. This makes sense because
# these builds of the libraries are only used by GCC, so it
Expand All @@ -398,21 +437,15 @@ in
})


# Construct a fourth stdenv that uses the new GCC. But coreutils is
# Construct a fifth stdenv that uses the new GCC. But coreutils is
# still from the bootstrap tools.
#
# resulting stage4 stdenv:
# resulting stage5 stdenv:
# - coreutils: from bootstrapFiles
# - glibc, binutils: from nixpkgs, built by bootstrapFiles toolchain
# - gcc: from nixpkgs, built by bootstrapFiles toolchain. Can assume
# it has almost no code from bootstrapTools as gcc bootstraps
# internally. The only exceptions are crt files from glibc
# built by bootstrapTools used to link executables and libraries,
# and the bootstrapTools-built, statically-linked
# lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc
# (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d).
# - binutils: from nixpkgs, built by bootstrapFiles toolchain
# - glibc, gcc: from nixpkgs, built by nixpkgs toolchain
(prevStage: stageFun prevStage {
name = "bootstrap-stage4";
name = "bootstrap-stage5";

overrides = self: super: {
# Zlib has to be inherited and not rebuilt in this stage,
Expand All @@ -433,7 +466,7 @@ in
# force gmp to rebuild so we have the option of dynamically linking
# libgmp without creating a reference path from:
# stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap
gmp = lib.makeOverridable (super.gmp.override { stdenv = self.stdenv; }).overrideAttrs (a: { pname = "${a.pname}-stage4"; });
gmp = lib.makeOverridable (super.gmp.override { stdenv = self.stdenv; }).overrideAttrs (a: { pname = "${a.pname}-stage5"; });

# To allow users' overrides inhibit dependencies too heavy for
# bootstrap, like guile: https://github.com/NixOS/nixpkgs/issues/181188
Expand Down Expand Up @@ -469,15 +502,7 @@ in
# binutils built.
#
# resulting stage5 (final) stdenv:
# - coreutils, binutils: from nixpkgs, built by nixpkgs toolchain
# - glibc: from nixpkgs, built by bootstrapFiles toolchain
# - gcc: from nixpkgs, built by bootstrapFiles toolchain. Can assume
# it has almost no code from bootstrapTools as gcc bootstraps
# internally. The only exceptions are crt files from glibc
# built by bootstrapTools used to link executables and libraries,
# and the bootstrapTools-built, statically-linked
# lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc
# (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d).
# - coreutils, binutils, glibc, gcc: from nixpkgs, built by nixpkgs toolchain
(prevStage: {
inherit config overlays;
stdenv = import ../generic rec {
Expand Down

0 comments on commit 8ee2b7c

Please sign in to comment.