|
| 1 | +#===-----------------------------------------------------------------------===// |
| 2 | +# |
| 3 | +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | +# See https://llvm.org/LICENSE.txt for license information. |
| 5 | +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | +# |
| 7 | +#===----------------------------------------------------------------------===// |
| 8 | +# |
| 9 | +# This is a Nix recipe for collecting reproducers for benchmarking purposes in a |
| 10 | +# reproducible way. It works by injecting a linker wrapper that embeds a |
| 11 | +# reproducer tarball into a non-allocated section of every linked object, which |
| 12 | +# generally causes them to be smuggled out of the build tree in a section of the |
| 13 | +# final binaries. In principle, this technique should let us collect reproducers |
| 14 | +# from any project packaged by Nix without project-specific knowledge, but as |
| 15 | +# you can see below, many interesting ones need a few hacks. |
| 16 | +# |
| 17 | +# If you have Nix installed, you can build the reproducers with the following |
| 18 | +# command: |
| 19 | +# |
| 20 | +# TMPDIR=/var/tmp nix-build -j6 --log-format bar collect.nix |
| 21 | +# |
| 22 | +# This will result in building several large projects including Chromium and |
| 23 | +# Firefox, which will take some time, and it will also build most of the |
| 24 | +# dependencies for non-native targets. Eventually you will get a result |
| 25 | +# directory containing all the reproducers. |
| 26 | +# |
| 27 | +# The following projects have been tested successfully: |
| 28 | +# - chrome (native only, cross builds fail building the qtbase dependency) |
| 29 | +# - firefox (all targets) |
| 30 | +# - linux-kernel (all targets, requires patched nixpkgs) |
| 31 | +# - ladybird (native only, same problem as chromium) |
| 32 | +# - llvm (all targets) |
| 33 | + |
| 34 | +{ |
| 35 | + nixpkgs ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/992f916556fcfaa94451ebc7fc6e396134bbf5b1.tar.gz", |
| 36 | + system ? builtins.currentSystem, |
| 37 | +}: |
| 38 | +let |
| 39 | + reproducerPkgs = |
| 40 | + crossSystem: |
| 41 | + let |
| 42 | + pkgsCross = import nixpkgs { inherit crossSystem; }; |
| 43 | + # Wraps the given stdenv and lld package into a variant that collects |
| 44 | + # the reproducer and builds with debug info. |
| 45 | + reproducerCollectingStdenv = |
| 46 | + stdenv: lld: |
| 47 | + let |
| 48 | + bintools = stdenv.cc.bintools.override { |
| 49 | + extraBuildCommands = '' |
| 50 | + wrap ${stdenv.cc.targetPrefix}nix-wrap-lld ${pkgsCross.path}/pkgs/build-support/bintools-wrapper/ld-wrapper.sh ${lld}/bin/ld.lld |
| 51 | + export lz4=${pkgsCross.lib.getBin pkgsCross.buildPackages.lz4}/bin/lz4 |
| 52 | + substituteAll ${./ld-wrapper.sh} $out/bin/${stdenv.cc.targetPrefix}ld |
| 53 | + chmod +x $out/bin/${stdenv.cc.targetPrefix}ld |
| 54 | + substituteAll ${./ld-wrapper.sh} $out/bin/${stdenv.cc.targetPrefix}ld.lld |
| 55 | + chmod +x $out/bin/${stdenv.cc.targetPrefix}ld.lld |
| 56 | + ''; |
| 57 | + }; |
| 58 | + in |
| 59 | + pkgsCross.withCFlags [ "-g1" ] ( |
| 60 | + stdenv.override (old: { |
| 61 | + allowedRequisites = null; |
| 62 | + cc = stdenv.cc.override { inherit bintools; }; |
| 63 | + }) |
| 64 | + ); |
| 65 | + withReproducerCollectingStdenv = |
| 66 | + pkg: |
| 67 | + pkg.override { |
| 68 | + stdenv = reproducerCollectingStdenv pkgsCross.stdenv pkgsCross.buildPackages.lld; |
| 69 | + }; |
| 70 | + withReproducerCollectingClangStdenv = |
| 71 | + pkg: |
| 72 | + pkg.override { |
| 73 | + clangStdenv = reproducerCollectingStdenv pkgsCross.clangStdenv pkgsCross.buildPackages.lld; |
| 74 | + }; |
| 75 | + in |
| 76 | + { |
| 77 | + # For benchmarking the linker we want to disable LTO as otherwise we would |
| 78 | + # just be benchmarking the LLVM optimizer. Also, we generally want the |
| 79 | + # package to use the regular stdenv in order to simplify wrapping it. |
| 80 | + # Firefox normally uses the rustc stdenv but uses the regular one if |
| 81 | + # LTO is disabled so we kill two birds with one stone by disabling it. |
| 82 | + # Chromium uses the rustc stdenv unconditionally so we need the stuff |
| 83 | + # below to make sure that it finds our wrapped stdenv. |
| 84 | + chrome = |
| 85 | + (pkgsCross.chromium.override { |
| 86 | + newScope = |
| 87 | + extra: |
| 88 | + pkgsCross.newScope ( |
| 89 | + extra |
| 90 | + // { |
| 91 | + pkgsBuildBuild = { |
| 92 | + pkg-config = pkgsCross.pkgsBuildBuild.pkg-config; |
| 93 | + rustc = { |
| 94 | + llvmPackages = rec { |
| 95 | + stdenv = reproducerCollectingStdenv pkgsCross.pkgsBuildBuild.rustc.llvmPackages.stdenv pkgsCross.pkgsBuildBuild.rustc.llvmPackages.lld; |
| 96 | + bintools = stdenv.cc.bintools; |
| 97 | + }; |
| 98 | + }; |
| 99 | + }; |
| 100 | + } |
| 101 | + ); |
| 102 | + pkgs = { |
| 103 | + rustc = { |
| 104 | + llvmPackages = { |
| 105 | + stdenv = reproducerCollectingStdenv pkgsCross.rustc.llvmPackages.stdenv pkgsCross.rustc.llvmPackages.lld; |
| 106 | + }; |
| 107 | + }; |
| 108 | + }; |
| 109 | + }).browser.overrideAttrs |
| 110 | + (old: { |
| 111 | + configurePhase = old.configurePhase + '' |
| 112 | + echo use_thin_lto = false >> out/Release/args.gn |
| 113 | + echo is_cfi = false >> out/Release/args.gn |
| 114 | + ''; |
| 115 | + }); |
| 116 | + firefox = (withReproducerCollectingStdenv pkgsCross.firefox-unwrapped).override { |
| 117 | + ltoSupport = false; |
| 118 | + pgoSupport = false; |
| 119 | + }; |
| 120 | + # Doesn't work as-is because the kernel derivation calls the linker |
| 121 | + # directly instead of the wrapper. See: |
| 122 | + # https://github.com/NixOS/nixpkgs/blob/d3fdff1631946f3e51318317375d638dae3d6aa2/pkgs/os-specific/linux/kernel/common-flags.nix#L12 |
| 123 | + linux-kernel = (withReproducerCollectingStdenv pkgsCross.linux_latest).dev; |
| 124 | + ladybird = withReproducerCollectingStdenv pkgsCross.ladybird; |
| 125 | + llvm = withReproducerCollectingStdenv pkgsCross.llvm; |
| 126 | + webkitgtk = withReproducerCollectingClangStdenv pkgsCross.webkitgtk; |
| 127 | + hello = withReproducerCollectingStdenv pkgsCross.hello; |
| 128 | + }; |
| 129 | + targets = { |
| 130 | + x86_64 = reproducerPkgs { config = "x86_64-unknown-linux-gnu"; }; |
| 131 | + aarch64 = reproducerPkgs { config = "aarch64-unknown-linux-gnu"; }; |
| 132 | + riscv64 = reproducerPkgs { config = "riscv64-unknown-linux-gnu"; }; |
| 133 | + }; |
| 134 | + pkgs = import nixpkgs { }; |
| 135 | +in |
| 136 | +pkgs.runCommand "lld-speed-test" { } '' |
| 137 | + extract_reproducer() { |
| 138 | + ${pkgs.coreutils}/bin/mkdir -p $out/$2 |
| 139 | + ${pkgs.llvm}/bin/llvm-objcopy -O binary --only-section=.lld_repro --set-section-flags .lld_repro=alloc $1 - | ${pkgs.gnutar}/bin/tar x -I ${pkgs.lib.getBin pkgs.buildPackages.lz4}/bin/lz4 --strip-components=1 -C $out/$2 |
| 140 | + } |
| 141 | +
|
| 142 | + extract_reproducer ${targets.aarch64.hello}/bin/hello hello-arm64 |
| 143 | + extract_reproducer ${targets.x86_64.hello}/bin/hello hello-x64 |
| 144 | + extract_reproducer ${targets.aarch64.chrome}/libexec/chromium/chromium chrome |
| 145 | + extract_reproducer ${targets.aarch64.ladybird}/lib/liblagom-web.so ladybird |
| 146 | + extract_reproducer ${targets.aarch64.firefox}/lib/firefox/libxul.so firefox-arm64 |
| 147 | + extract_reproducer ${targets.x86_64.firefox}/lib/firefox/libxul.so firefox-x64 |
| 148 | + extract_reproducer ${targets.riscv64.firefox}/lib/firefox/libxul.so firefox-riscv64 |
| 149 | + extract_reproducer ${pkgs.lib.getLib targets.aarch64.llvm}/lib/libLLVM.so llvm-arm64 |
| 150 | + extract_reproducer ${pkgs.lib.getLib targets.x86_64.llvm}/lib/libLLVM.so llvm-x64 |
| 151 | + extract_reproducer ${pkgs.lib.getLib targets.riscv64.llvm}/lib/libLLVM.so llvm-riscv6 |
| 152 | +'' |
0 commit comments