Add tarballs with static binaries (#2025)

* Add tarballs with static binaries

* Expose nix-tools hydraJobs in the main flake

* Pass pkgs to flake-compat

* Move exes into a bin sub dir

* Use static nix-tools

* Use static nix-tools

* Clearer nix-tools tarball code

* Add cabal and hpack to the nix-tools tarball

* Simplify nix-tools tarball

* Update nix-tools tarball

* Use cabal-install from nix-tools tarball

* Remove nix-tools wrapper script

* New clothes for nix-tools's tarball

* Temporarily disable the tarball

* Update nix-tools tarball and s/attrValues/attrNames

* Add s/internal-nix-tools/unchecked-nix-tools

* Add s/unchecked-nix-tools/nix-tools-unchecked

* Get `cabal-install` from `nix-tools`

* Qualify the nix-tools tarball with host platform

* Add comments

* Bump nix-tools tarball

---------

Co-authored-by: Hamish Mackenzie <Hamish.K.Mackenzie@gmail.com>

Author: andreabedini

build.nix

@@ -74,15 +74,15 @@ in rec {
       inherit (pkgs) stdenv lib writeScript coreutils glibc git
         openssh nixFlakes gawk bash curl findutils;
       # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
-      nix-tools = haskell.internal-nix-tools;
+      nix-tools = haskell.nix-tools-unchecked;
       cabal-install = haskell.internal-cabal-install;
       inherit (haskell) update-index-state-hashes cabal-issue-8352-workaround;
     };
     update-stackage = haskell.callPackage ./scripts/update-stackage.nix {
       inherit (pkgs) stdenv lib writeScript coreutils glibc git
         openssh nixFlakes gawk bash curl findutils;
       # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
-      nix-tools = haskell.internal-nix-tools;
+      nix-tools = haskell.nix-tools-unchecked;
       cabal-install = haskell.internal-cabal-install;
       inherit (haskell) cabal-issue-8352-workaround;
     };

builder/comp-builder.nix

@@ -294,10 +294,8 @@ let
         (if cabalFile != null
           then ''cp -v $cabalFilePath ${package.identifier.name}.cabal''
           else
-            # When building hpack package we use the internal nix-tools
-            # (compiled with a fixed GHC version)
             lib.optionalString (cabal-generator == "hpack") ''
-              ${buildPackages.haskell-nix.internal-nix-tools}/bin/hpack
+              ${buildPackages.haskell-nix.nix-tools-unchecked}/bin/hpack
             ''
         ) + lib.optionalString (prePatch != null) "\n${prePatch}";
     }

flake.nix

@@ -66,8 +66,11 @@
     };
   };
 
-  outputs = { self, nixpkgs, nixpkgs-unstable, nixpkgs-2105, nixpkgs-2111, nixpkgs-2205, nixpkgs-2211, nixpkgs-2305, flake-utils, ... }@inputs:
-    let compiler = "ghc928";
+  outputs = { self, nixpkgs, nixpkgs-unstable, nixpkgs-2105, nixpkgs-2111, nixpkgs-2205, nixpkgs-2211, nixpkgs-2305, flake-compat, flake-utils, ... }@inputs:
+    let
+      callFlake = import flake-compat;
+
+      compiler = "ghc928";
       config = import ./config.nix;
 
       traceNames = prefix: builtins.mapAttrs (n: v:
@@ -79,11 +82,20 @@
 
       traceHydraJobs = x: x // { inherit (traceNames "" x) hydraJobs; };
 
+      # systems supported by haskell.nix
+      systems = [
+        "x86_64-linux"
+        "x86_64-darwin"
+        # TODO switch back on when ci.iog.io has builders for aarch64-linux
+        # "aarch64-linux"
+        "aarch64-darwin"
+      ];
+
     in traceHydraJobs ({
       inherit config;
       overlay = self.overlays.combined;
       overlays = import ./overlays { sources = inputs; };
-      internal = rec {
+      internal = {
         nixpkgsArgs = {
           inherit config;
           overlays = [ self.overlay ];
@@ -139,85 +151,93 @@
       # supported by haskell.nix, e.g. with remote builders, in order to check this flake.
       # If you want to run the tests for just your platform, run `./test/tests.sh` or
       # `nix-build -A checks.$PLATFORM`
-    } // flake-utils.lib.eachSystem [
-        "x86_64-linux"
-        "x86_64-darwin"
-        # TODO switch back on when ci.iog.io has builders for aarch64-linux
-        # "aarch64-linux"
-        "aarch64-darwin" ] (system: rec {
-      legacyPackages = (self.internal.compat { inherit system; }).pkgs;
-      legacyPackagesUnstable = (self.internal.compat { inherit system; }).pkgs-unstable;
-
-      # FIXME: Currently `nix flake check` requires `--impure` because coverage-golden
-      # (and maybe other tests) import projects that use builtins.currentSystem
-      checks = builtins.listToAttrs (map (pkg: {
-        name = pkg.name;
-        value = pkg;
-      }) (nixpkgs.lib.collect nixpkgs.lib.isDerivation (import ./test rec {
-        haskellNix = self.internal.compat { inherit system; };
-        compiler-nix-name = compiler;
-        pkgs = haskellNix.pkgs;
-      })));
-      # Exposed so that buildkite can check that `allow-import-from-derivation=false` works for core of haskell.nix
-      roots = legacyPackagesUnstable.haskell-nix.roots compiler;
-
-      packages = (self.internal.compat { inherit system; }).hix.apps;
-
-      allJobs =
-        let
-          inherit (import ./ci-lib.nix { pkgs = legacyPackagesUnstable; }) stripAttrsForHydra filterDerivations;
-          ci = import ./ci.nix { inherit (self.internal) compat; inherit system; };
-        in stripAttrsForHydra (filterDerivations ci);
-
-      requiredJobs =
-        let
-          inherit (legacyPackages) lib;
-          names = x: lib.filter (n: n != "recurseForDerivations" && n != "meta")
-              (builtins.attrNames x);
-        in
-          builtins.listToAttrs (
-              lib.concatMap (nixpkgsVer:
-                let nixpkgsJobs = allJobs.${nixpkgsVer};
-                in lib.concatMap (compiler-nix-name:
-                  let ghcJobs = nixpkgsJobs.${compiler-nix-name};
-                  in builtins.map (crossPlatform: {
-                      name = "required-${nixpkgsVer}-${compiler-nix-name}-${crossPlatform}";
-                      value = legacyPackages.releaseTools.aggregate {
-                        name = "haskell.nix-${nixpkgsVer}-${compiler-nix-name}-${crossPlatform}";
-                        meta.description = "All ${nixpkgsVer} ${compiler-nix-name} ${crossPlatform} jobs";
-                        constituents = lib.collect lib.isDerivation ghcJobs.${crossPlatform};
-                      };
-                   }) (names ghcJobs)
-                ) (names nixpkgsJobs)
-              ) (names allJobs));
-
-      hydraJobs = allJobs;
-
-      devShells = with self.legacyPackages.${system}; {
-        default =
-          mkShell {
-            buildInputs = [
-              nixUnstable
-              cabal-install
-              haskell-nix.compiler.${compiler}
-            ];
-          };
-      } // __mapAttrs (compiler-nix-name: compiler:
-          mkShell {
-            buildInputs = [
-              compiler
-              haskell-nix.cabal-install.${compiler-nix-name}
-            ];
-          }
-      ) (
-        # Exclude old versions of GHC to speed up `nix flake check`
-        builtins.removeAttrs haskell-nix.compiler
-          [ "ghc844"
-            "ghc861" "ghc862" "ghc863" "ghc864"
-            "ghc881" "ghc882" "ghc883"
-            "ghc8101" "ghc8102" "ghc8103" "ghc8104" "ghc8105" "ghc8106" "ghc810420210212"
-            "ghc901"
-            "ghc921" "ghc922" "ghc923"]);
+    } // flake-utils.lib.eachSystem systems (system:
+      let
+        legacyPackages = (self.internal.compat { inherit system; }).pkgs;
+        nix-tools-hydraJobs =
+          let cf = callFlake { pkgs = legacyPackages; inherit system; src = ./nix-tools; };
+          in cf.defaultNix.hydraJobs;
+      in rec {
+        inherit legacyPackages;
+        legacyPackagesUnstable = (self.internal.compat { inherit system; }).pkgs-unstable;
+
+        # FIXME: Currently `nix flake check` requires `--impure` because coverage-golden
+        # (and maybe other tests) import projects that use builtins.currentSystem
+        checks = builtins.listToAttrs (map (pkg: {
+          name = pkg.name;
+          value = pkg;
+        }) (nixpkgs.lib.collect nixpkgs.lib.isDerivation (import ./test rec {
+          haskellNix = self.internal.compat { inherit system; };
+          compiler-nix-name = compiler;
+          pkgs = haskellNix.pkgs;
+        })));
+        # Exposed so that buildkite can check that `allow-import-from-derivation=false` works for core of haskell.nix
+        roots = legacyPackagesUnstable.haskell-nix.roots compiler;
+
+        packages = (self.internal.compat { inherit system; }).hix.apps;
+
+        allJobs =
+          let
+            inherit (import ./ci-lib.nix { pkgs = legacyPackagesUnstable; }) stripAttrsForHydra filterDerivations;
+            ci = import ./ci.nix { inherit (self.internal) compat; inherit system; };
+          in stripAttrsForHydra (filterDerivations ci);
+
+        requiredJobs =
+          let
+            inherit (legacyPackages) lib;
+            names = x: lib.filter (n: n != "recurseForDerivations" && n != "meta")
+                (builtins.attrNames x);
+          in
+            builtins.listToAttrs (
+                lib.concatMap (nixpkgsVer:
+                  let nixpkgsJobs = allJobs.${nixpkgsVer};
+                  in lib.concatMap (compiler-nix-name:
+                    let ghcJobs = nixpkgsJobs.${compiler-nix-name};
+                    in builtins.map (crossPlatform: {
+                        name = "required-${nixpkgsVer}-${compiler-nix-name}-${crossPlatform}";
+                        value = legacyPackages.releaseTools.aggregate {
+                          name = "haskell.nix-${nixpkgsVer}-${compiler-nix-name}-${crossPlatform}";
+                          meta.description = "All ${nixpkgsVer} ${compiler-nix-name} ${crossPlatform} jobs";
+                          constituents = lib.collect lib.isDerivation ghcJobs.${crossPlatform};
+                        };
+                    }) (names ghcJobs)
+                  ) (names nixpkgsJobs)
+                ) (names allJobs));
+
+        hydraJobs =
+          allJobs
+          # Include hydraJobs from nix-tools subflake.
+          # NOTE: These derivations do not depend on the haskell.nix in ./. but
+          # on the version of haskell.nix locked in the subflake. They are
+          # evaluated within their own flake and independently of anything
+          # else. Here we only expose them in the main flake.
+          // { nix-tools = nix-tools-hydraJobs.${system} or {}; };
+
+        devShells = with self.legacyPackages.${system}; {
+          default =
+            mkShell {
+              buildInputs = [
+                nixUnstable
+                cabal-install
+                haskell-nix.compiler.${compiler}
+              ];
+            };
+        } // __mapAttrs (compiler-nix-name: compiler:
+            mkShell {
+              buildInputs = [
+                compiler
+                haskell-nix.cabal-install.${compiler-nix-name}
+              ];
+            }
+        ) (
+          # Exclude old versions of GHC to speed up `nix flake check`
+          builtins.removeAttrs haskell-nix.compiler
+            [ "ghc844"
+              "ghc861" "ghc862" "ghc863" "ghc864"
+              "ghc881" "ghc882" "ghc883"
+              "ghc8101" "ghc8102" "ghc8103" "ghc8104" "ghc8105" "ghc8106" "ghc810420210212"
+              "ghc901"
+              "ghc921" "ghc922" "ghc923"]);
     }));
 
   # --- Flake Local Nix Configuration ----------------------------

lib/call-cabal-project-to-nix.nix

@@ -73,9 +73,7 @@ let
   nix-tools = if args.nix-tools or null != null
     then args.nix-tools
     else evalPackages.haskell-nix.nix-tools-unchecked;
-  cabal-install = if args.cabal-install or null != null
-    then args.cabal-install
-    else evalPackages.haskell-nix.cabal-install-unchecked.${compiler-nix-name};
+  cabal-install = nix-tools.exes.cabal;
   forName = pkgs.lib.optionalString (name != null) (" for " + name);
   nameAndSuffix = suffix: if name == null then suffix else name + "-" + suffix;
 
@@ -368,11 +366,14 @@ let
   } // pkgs.lib.optionalAttrs (checkMaterialization != null) {
     inherit checkMaterialization;
   }) (evalPackages.runCommand (nameAndSuffix "plan-to-nix-pkgs") {
-    nativeBuildInputs = [
-      nix-tools.exes.make-install-plan
-      nix-tools.exes.plan-to-nix
-      dummy-ghc dummy-ghc-pkg cabal-install evalPackages.rsync evalPackages.gitMinimal evalPackages.allPkgConfigWrapper ]
-      ++ pkgs.lib.optional supportHpack nix-tools.exes.hpack;
+    nativeBuildInputs =
+      # The things needed from nix-tools
+      [ nix-tools.exes.make-install-plan
+        nix-tools.exes.plan-to-nix
+        nix-tools.exes.cabal
+      ]
+      ++ pkgs.lib.optional supportHpack nix-tools.exes.hpack
+      ++ [dummy-ghc dummy-ghc-pkg evalPackages.rsync evalPackages.gitMinimal evalPackages.allPkgConfigWrapper ];
     # Needed or stack-to-nix will die on unicode inputs
     LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
     LANG = "en_US.UTF-8";

modules/cabal-project.nix

@@ -90,11 +90,6 @@ in {
       default = null;
       description = "nix-tools to use when converting the `plan.json` to nix";
     };
-    cabal-install = mkOption {
-      type = nullOr package;
-      default = null;
-      description = "cabal-install to use when running `cabal configure`";
-    };
     configureArgs = mkOption {
       type = nullOr (separatedString " ");
       default = "";

modules/stack-project.nix

@@ -37,7 +37,7 @@ with types;
     };
     nix-tools = mkOption {
       type = package;
-      default = config.evalPackages.haskell-nix.internal-nix-tools; # When building stack projects we use the internal nix-tools (compiled with a fixed GHC version)
+      default = config.evalPackages.haskell-nix.nix-tools-unchecked; # When building stack projects we use the internal nix-tools (compiled with a fixed GHC version)
       description = "nix-tools to use when converting the `plan.json` to nix";
     };
 

nix-tools/cabal.project

@@ -4,6 +4,11 @@ with-compiler: ghc-9.2.8
 
 packages: nix-tools
 
+-- haskell.nix expects nix-tools to provide the cabal and hpack executables
+-- so we put these two packages here, so they will be present even if nix-tools
+-- dependencies change
+extra-packages: cabal-install, hpack
+
 test-show-details: direct
 
 allow-newer: