Merge pull request #83 from yusdacra/rust/crane-builder

feat(rust): implement crane builder
nix-community · Feb 28, 2022 · 4656834 · 4656834
2 parents e12bc2f + c580c24
commit 4656834
Show file tree

Hide file tree

Showing 14 changed files with 382 additions and 18 deletions.
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -15,6 +15,9 @@
 
     # required for utils.satisfiesSemver
     poetry2nix = { url = "github:nix-community/poetry2nix/1.21.0"; flake = false; };
+
+    # required for builder rust/crane
+    crane = { url = "github:ipetkov/crane"; flake = false; };
   };
 
   outputs = {
@@ -24,6 +27,7 @@
     nixpkgs,
     node2nix,
     poetry2nix,
+    crane,
   }@inp:
     let
 
@@ -60,6 +64,21 @@
           "semver.nix"
           "LICENSE"
         ];
+        crane = [
+          "lib/buildDepsOnly.nix"
+          "lib/cargoBuild.nix"
+          "lib/buildPackage.nix"
+          "lib/mkCargoDerivation.nix"
+          "lib/mkDummySrc.nix"
+          "lib/writeTOML.nix"
+          "lib/cleanCargoToml.nix"
+          "pkgs/configureCargoCommonVarsHook.sh"
+          "pkgs/configureCargoVendoredDepsHook.sh"
+          "pkgs/inheritCargoArtifactsHook.sh"
+          "pkgs/installCargoArtifactsHook.sh"
+          "pkgs/remapSourcePathPrefixHook.sh"
+          "LICENSE"
+        ];
       };
 
       # create a directory containing the files listed in externalPaths

diff --git a/src/builders/default.nix b/src/builders/default.nix
@@ -24,5 +24,8 @@
     default = buildRustPackage;
 
     buildRustPackage = callPackageDream ./rust/build-rust-package {};
+
+    # this builder requires IFD!
+    crane = callPackageDream ./rust/crane {};
   };
 }
diff --git a/src/builders/rust/build-rust-package/default.nix b/src/builders/rust/build-rust-package/default.nix
@@ -22,25 +22,28 @@
 let
   l = lib // builtins;
 
-  vendoring = import ../vendor.nix {
-    inherit lib pkgs getSource getSourceSpec
-    getDependencies getCyclicDependencies subsystemAttrs;
-  };
+  utils = import ../utils.nix args;
+  vendoring = import ../vendor.nix (args // { inherit lib pkgs utils; });
 
   buildPackage = pname: version:
     let
-      src = getSource pname version;
-      vendorDir = vendoring.vendorPackageDependencies pname version;
+      src = utils.getRootSource pname version;
+      vendorDir = vendoring.vendorDependencies pname version;
+
+      cargoBuildFlags = "--package ${pname}";
     in
     produceDerivation pname (pkgs.rustPlatform.buildRustPackage {
       inherit pname version src;
 
+      cargoBuildFlags = cargoBuildFlags;
+      cargoCheckFlags = cargoBuildFlags;
+
+      cargoVendorDir = "../nix-vendor";
+
       postUnpack = ''
         ln -s ${vendorDir} ./nix-vendor
       '';
 
-      cargoVendorDir = "../nix-vendor";
-
       preBuild = ''
         ${vendoring.writeGitVendorEntries "vendored-sources"}
       '';

diff --git a/src/builders/rust/crane/default.nix b/src/builders/rust/crane/default.nix
@@ -0,0 +1,61 @@
+{
+  lib,
+  pkgs,
+
+  externals,
+  ...
+}:
+
+{
+  subsystemAttrs,
+  defaultPackageName,
+  defaultPackageVersion,
+  getCyclicDependencies,
+  getDependencies,
+  getSource,
+  getSourceSpec,
+  packages,
+  produceDerivation,
+
+  ...
+}@args:
+
+let
+  l = lib // builtins;
+
+  utils = import ../utils.nix args;
+  vendoring = import ../vendor.nix (args // { inherit lib pkgs utils; });
+
+  crane = externals.crane;
+
+  buildPackage = pname: version:
+    let
+      src = utils.getRootSource pname version;
+      cargoVendorDir = vendoring.vendorDependencies pname version;
+      preBuild = ''
+        ${vendoring.writeGitVendorEntries "nix-sources"}
+      '';
+      # The deps-only derivation will use this as a prefix to the `pname`
+      depsNameSuffix = "-deps";
+      # Make sure cargo only builds the package we want
+      cargoExtraArgs = "--package ${pname}";
+
+      deps = produceDerivation "${pname}${depsNameSuffix}" (crane.buildDepsOnly {
+        inherit pname version src cargoVendorDir cargoExtraArgs preBuild;
+        pnameSuffix = depsNameSuffix;
+      });
+    in
+    produceDerivation pname (crane.buildPackage {
+      inherit pname version src cargoVendorDir cargoExtraArgs preBuild;
+      cargoArtifacts = deps;
+    });
+in
+rec {
+  packages =
+    l.mapAttrs
+      (name: version:
+        { "${version}" = buildPackage name version; })
+      args.packages;
+
+  defaultPackage = packages."${defaultPackageName}"."${defaultPackageVersion}";
+}
diff --git a/src/builders/rust/utils.nix b/src/builders/rust/utils.nix
@@ -0,0 +1,12 @@
+{
+  getSourceSpec,
+  getSource,
+  getRoot,
+
+  ...
+}:
+rec {
+  getRootSource = pname: version:
+    let root = getRoot pname version; in
+    getSource root.pname root.version;
+}
diff --git a/src/builders/rust/vendor.nix b/src/builders/rust/vendor.nix
@@ -2,11 +2,14 @@
   lib,
   pkgs,
 
+  getRoot,
   getSource,
   getSourceSpec,
   getDependencies,
   getCyclicDependencies,
   subsystemAttrs,
+
+  ...
 }:
 let
   l = lib // builtins;
@@ -38,7 +41,7 @@ let
         (l.filter (isCyclic cyclic) direct)
       )
     ));
-in {
+in rec {
   # Generates a shell script that writes git vendor entries to .cargo/config.
   # `replaceWith` is the name of the vendored source(s) to use.
   writeGitVendorEntries = replaceWith:
@@ -122,4 +125,9 @@ in {
         sources
        }
     '';
-}
+
+  # Vendors a package's roots dependencies.
+  vendorDependencies = pname: version:
+    let root = getRoot pname version; in
+    vendorPackageDependencies root.pname root.version;
+}
diff --git a/src/default.nix b/src/default.nix
@@ -98,6 +98,76 @@ let
       pkgs.callPackage "${externalSources.node2nix}/nix/node-env.nix" {
         inherit nodejs;
       };
+    crane =
+      let
+        importLibFile = name: import "${externalSources.crane}/lib/${name}.nix";
+
+        makeHook = attrs: name:
+          pkgs.makeSetupHook
+            ({ inherit name; } // attrs)
+            "${externalSources.crane}/pkgs/${name}.sh";
+        genHooks = names: attrs: lib.genAttrs names (makeHook attrs);
+
+        otherHooks =
+          genHooks [
+            "configureCargoCommonVarsHook"
+            "configureCargoVendoredDepsHook"
+            "remapSourcePathPrefixHook"
+          ] { };
+        installHooks =
+          genHooks [
+            "inheritCargoArtifactsHook"
+            "installCargoArtifactsHook"
+          ] {
+            substitutions = {
+              zstd = "${pkgs.pkgsBuildBuild.zstd}/bin/zstd";
+            };
+          };
+        installLogHook = genHooks ["installFromCargoBuildLogHook"] {
+          substitutions = {
+            cargo = "${pkgs.pkgsBuildBuild.cargo}/bin/cargo";
+            jq = "${pkgs.pkgsBuildBuild.jq}/bin/jq";
+          };
+        };
+      in rec {
+        # These aren't used by dream2nix
+        crateNameFromCargoToml = null;
+        vendorCargoDeps = null;
+
+        writeTOML = importLibFile "writeTOML" {
+          inherit (pkgs) writeText;
+          inherit (utils) toTOML;
+        };
+        cleanCargoToml = importLibFile "cleanCargoToml" {
+          inherit (builtins) fromTOML;
+        };
+        findCargoFiles = importLibFile "findCargoFiles" {
+          inherit (pkgs) lib;
+        };
+        mkDummySrc = importLibFile "mkDummySrc" {
+          inherit (pkgs) writeText runCommandLocal lib;
+          inherit writeTOML cleanCargoToml findCargoFiles;
+        };
+
+        mkCargoDerivation = importLibFile "mkCargoDerivation" ({
+          inherit (pkgs) cargo stdenv lib;
+        } // installHooks // otherHooks);
+        buildDepsOnly = importLibFile "buildDepsOnly" {
+          inherit
+            mkCargoDerivation crateNameFromCargoToml
+            vendorCargoDeps mkDummySrc;
+        };
+        cargoBuild = importLibFile "cargoBuild" {
+          inherit
+            mkCargoDerivation buildDepsOnly
+            crateNameFromCargoToml vendorCargoDeps;
+        };
+        buildPackage = importLibFile "buildPackage" {
+          inherit (pkgs) lib;
+          inherit (installLogHook) installFromCargoBuildLogHook;
+          inherit cargoBuild;
+        };
+      };
   };
 
   dreamOverrides =
@@ -309,6 +379,7 @@ let
           inherit (dreamLockInterface)
             subsystemAttrs
             getSourceSpec
+            getRoot
             getDependencies
             getCyclicDependencies
             defaultPackageName

diff --git a/src/lib.nix b/src/lib.nix
@@ -110,6 +110,10 @@ let
             (system: pkgs:
               dream2nixFor."${system}".apps.flakeApps.dream2nix);
 
+        builders =
+          forAllSystems
+            (system: pkgs:
+              dream2nixFor."${system}".builders);
       };
 
   makeFlakeOutputsFunc =

diff --git a/src/specifications/dream-lock-schema.json b/src/specifications/dream-lock-schema.json
@@ -96,6 +96,8 @@
                   },
                   "then": {
                     "properties": {
+                      "rootName": { "type": "string" },
+                      "rootVersion": { "type": "string" },
                       "path": { "type": "string" },
                       "type": { "type": "string" },
                       "dir": { "type": "string" }

diff --git a/src/translators/rust/pure/cargo-lock/default.nix b/src/translators/rust/pure/cargo-lock/default.nix
@@ -63,18 +63,18 @@ in
                 || l.pathExists "${l.dirOf toml.path}/src/bin";
 
             # Try to find a package with a binary
-            pkg =
-              l.findFirst
-              hasBinaries
-              (l.warn "couldn't find a package with a binary to use as mainPackage" (l.elemAt cargoPackages 0))
-              cargoPackages;
+            pkg = l.findFirst hasBinaries (l.elemAt cargoPackages 0) cargoPackages;
 
           in pkg.value.package.name
         else args.packageName;
 
       # Find the Cargo.toml matching the package name
       checkForPackageName = cargoToml: (cargoToml.value.package.name or null) == packageName;
-      packageToml = l.findFirst checkForPackageName (throw "no Cargo.toml found with the package name passed: ${packageName}") cargoTomls;
+      packageToml =
+        l.findFirst
+        checkForPackageName
+        (throw "no Cargo.toml found with the package name passed: ${packageName}")
+        cargoTomls;
 
       # Parse Cargo.lock and extract dependencies
       parsedLock = l.fromTOML (l.readFile "${inputDir}/Cargo.lock");
@@ -223,8 +223,9 @@ in
                 relDir = lib.removePrefix "${inputDir}/" (l.dirOf toml.path);
               in
               {
-                path =
-                  relDir;
+                path = relDir;
+                rootName = package.name;
+                rootVersion = package.version;
               };
 
             git = dependencyObject:

diff --git a/src/utils/default.nix b/src/utils/default.nix
@@ -65,6 +65,8 @@ rec {
 
   toDrv = path: runCommand "some-drv" {} "cp -r ${path} $out";
 
+  toTOML = import ./toTOML.nix { inherit lib; };
+
   # hash the contents of a path via `nix hash path`
   hashPath = algo: path:
     let