Merge pull request #18 from RyanGibb/main

`opam monorepo` support

README.md

@@ -582,6 +582,51 @@ Then, import it:
 Generate a nix attribute set from the opam file. This is just a Nix
 representation of the JSON produced by `opam2json`.
 
+### Monorepo functions
+
+#### `queryToMonorepo`
+
+```
+{ repos = ?[Repository]
+; resolveArgs = ?ResolveArgs
+; filterPkgs ?[ ] }
+→ Query
+→ Scope
+```
+
+Similar to `queryToScope`, but creates a attribute set (instead of a
+scope) with package names mapping to sources for replicating the
+[`opam monorepo`](https://github.com/tarides/opam-monorepo) workflow.
+
+The `filterPkgs` argument gives a list of package names to filter from
+the resulting attribute set, rather than removing them based on their
+opam `dev-repo` name.
+
+#### `buildOpamMonorepo`
+
+```
+{ repos = ?[Repository]
+; resolveArgs = ?ResolveArgs
+; pinDepends = ?Bool
+; recursive = ?Bool
+; extraFilterPkgs ?[ ] }
+→ project: Path
+→ Query
+→ Sources
+```
+
+A convenience wrapper around `queryToMonorepo`.
+
+Creates a monorepo for an opam project (found in the directory passed
+as the second argument). The monorepo consists of an attribute set of
+opam package `dev-repo`s to sources, for all dependancies of the
+packages found in the project directory as well as other packages from
+the `Query`.
+
+The packages in the project directory are excluded from
+the resulting monorepo along with `ocaml-system`, `opam-monorepo`, and
+packages in the `extraFilterPkgs` argument.
+
 ### Lower-level functions
 
 `joinRepos : [Repository] → Repository`
@@ -600,6 +645,12 @@ representation of the JSON produced by `opam2json`.
 
 `filterOpamRepo : Query → Repository → Repository`
 
+`defsToSrcs : Defs → Sources`
+
+`deduplicateSrcs : Sources → Sources`
+
+`mkMonorepo : Sources → Scope`
+
 `opamList` resolves package versions using the repo (first argument)
 and environment (second argument). Note that it accepts only one
 repo. If you want to pass multiple repositories, merge them together
@@ -626,6 +677,17 @@ packagedefs. Requires `--impure` (to fetch the repos specified in
 `filterOpamRepo` filters the repository to only include packages (and
 their particular versions) present in the supplied Query.
 
+`defsToSrcs` takes an attribute set of definitions (as produced by
+`queryToDefs`) and produces a list `Sources`
+( `[ { name; version; src; } ... ]`).
+
+`deduplicateSrcs` deduplicates `Sources` produced by `defsToSrcs`, as
+some packages may share sources if they are developed in the same repo.
+
+`mkMonorepo` takes `Sources` and creates an attribute set mapping
+package names to sources with a derivation that fetches the source
+at the `src` URL.
+
 #### `Defs` (set of package definitions)
 
 The attribute set of package definitions has package names as

flake.nix

@@ -14,10 +14,21 @@
       url = "github:ocaml/opam-repository";
       flake = false;
     };
+
+    # used for opam-monorepo
+    opam-overlays = {
+      url = "github:dune-universe/opam-overlays";
+      flake = false;
+    };
+    mirage-opam-overlays = {
+      url = "github:dune-universe/mirage-opam-overlays";
+      flake = false;
+    };
   };
 
   outputs =
-    { self, nixpkgs, flake-utils, opam2json, opam-repository, ... }@inputs:
+    { self, nixpkgs, flake-utils, opam2json, opam-repository, opam-overlays,
+      mirage-opam-overlays, ... }@inputs:
     {
       aux = import ./src/lib.nix nixpkgs.lib;
       templates.simple = {
@@ -46,7 +57,7 @@
             opam2json.overlay
             opam-overlay
           ]);
-        opam-nix = import ./src/opam.nix { inherit pkgs opam-repository; };
+        opam-nix = import ./src/opam.nix { inherit pkgs opam-repository opam-overlays mirage-opam-overlays; };
       in rec {
         lib = opam-nix;
         checks = packages

src/builder.nix

@@ -11,7 +11,7 @@ let
   inherit (import ./evaluator lib)
     setup compareVersions' collectAllValuesFromOptionList val functionArgsFor
     filterOptionList pkgVarsFor varsToShell filterSectionInShell normalize
-    normalize' getHashes envToShell;
+    normalize' getHashes envToShell getUrl;
 
   alwaysNative = import ./always-native.nix;
 
@@ -38,7 +38,7 @@ in { name, version, ... }@pkgdef: rec {
       inherit (deps.nixpkgs) stdenv;
       inherit (deps.nixpkgs.pkgsBuildBuild)
         envsubst writeText writeShellScriptBin writeShellScript unzip
-        emptyDirectory opam-installer jq opam2json removeReferencesTo;
+        opam-installer jq opam2json removeReferencesTo;
 
       globalVariables = import ./global-variables.nix stdenv.hostPlatform;
       # We have to resolve which packages we want at eval-time, except for with-test.
@@ -114,14 +114,6 @@ in { name, version, ... }@pkgdef: rec {
           (collectAllValuesFromOptionList
             (pkgdef.depends or [ ] ++ pkgdef.depopts or [ ]))));
 
-      hashes = if pkgdef.url ? checksum then
-        if isList pkgdef.url.checksum then
-          getHashes pkgdef.url.checksum
-        else
-          getHashes [ pkgdef.url.checksum ]
-      else
-        { };
-
       externalPackages = if (readDir ./overlays/external)
       ? "${globalVariables.os-distribution}.nix" then
         import (./overlays/external + "/${globalVariables.os-distribution}.nix")
@@ -142,15 +134,7 @@ in { name, version, ... }@pkgdef: rec {
         map (x: if isString (val x) then externalPackages.${val x} else null)
         extInputNames;
 
-      archive = pkgdef.url.src or pkgdef.url.archive or "";
-      src = if pkgdef ? url then
-      # Default unpacker doesn't support .zip
-        if hashes == { } then
-          builtins.fetchTarball archive
-        else
-          deps.nixpkgs.fetchurl ({ url = archive; } // hashes)
-      else
-        pkgdef.src or emptyDirectory;
+      inherit (getUrl deps.nixpkgs pkgdef) archive src;
 
       evalOpamVar = ''
         evalOpamVar() {

src/evaluator/default.nix

@@ -334,4 +334,25 @@ in rec {
   getHashes = checksums:
     head (concatMap (x: tryHash "sha512" x ++ tryHash "sha256" x ++ trymd5 x)
       checksums ++ [ { } ]);
+
+  getUrl = pkgs: pkgdef:
+    let
+      hashes = if pkgdef.url ? checksum then
+        if isList pkgdef.url.checksum then
+          getHashes pkgdef.url.checksum
+        else
+          getHashes [ pkgdef.url.checksum ]
+      else
+        { };
+      archive = pkgdef.url.src or pkgdef.url.archive or "";
+      src = if pkgdef ? url then
+      # Default unpacker doesn't support .zip
+        if hashes == { } then
+          builtins.fetchTarball archive
+        else
+          pkgs.fetchurl ({ url = archive; } // hashes)
+      else
+        pkgdef.src or pkgs.pkgsBuildBuild.emptyDirectory;
+    in { inherit archive src; };
+
 }

src/opam.nix

@@ -6,16 +6,17 @@ let
   inherit (builtins)
     readDir mapAttrs concatStringsSep isString isList attrValues filter head
     foldl' fromJSON listToAttrs readFile toFile isAttrs pathExists toJSON
-    deepSeq length sort concatMap attrNames;
+    deepSeq length sort concatMap attrNames path elem elemAt;
   bootstrapPackages = args.pkgs;
   inherit (bootstrapPackages) lib;
   inherit (lib)
     splitString tail nameValuePair zipAttrsWith collect concatLists
     filterAttrsRecursive fileContents pipe makeScope optionalAttrs hasSuffix
     converge mapAttrsRecursive composeManyExtensions removeSuffix optionalString
-    last init recursiveUpdate foldl optional optionals importJSON;
+    last init recursiveUpdate foldl optional optionals importJSON mapAttrsToList
+    remove findSingle filterAttrs hasInfix;
 
-  inherit (import ./evaluator lib) compareVersions';
+  inherit (import ./evaluator lib) compareVersions' getUrl collectAllValuesFromOptionList;
 
   readDirRecursive = dir:
     mapAttrs (name: type:
@@ -305,6 +306,8 @@ in rec {
   staticOverlay = import ./overlays/ocaml-static.nix;
   darwinOverlay = import ./overlays/ocaml-darwin.nix;
   opamRepository = args.opam-repository;
+  opamOverlays = args.opam-overlays;
+  mirageOpamOverlays = args.mirage-opam-overlays;
 
   __overlays = [
     (final: prev:
@@ -417,7 +420,7 @@ in rec {
             "[opam-nix] Nix version is too old for allRefs = true; fetching a repository may fail if the commit is on a non-master branch"
             { };
           path =
-            (builtins.fetchGit ({ inherit url; } // refsOrWarn // optionalRev))
+            (builtins.fetchGit ({ inherit url; submodules = true; } // refsOrWarn // optionalRev))
             // {
               inherit url;
             };
@@ -480,4 +483,103 @@ in rec {
         '';
       };
     in buildOpamProject args name generatedOpamFile query;
+
+  # takes an atribute set of package definitions (as produced by `queryToDefs`),
+  # deduplicates sources, and provides a list of sources to fetch
+  defsToSrcs = filterPkgs: defs:
+    let
+      # use our own version of lib.strings.nameFromURL without `assert name != filename`
+      nameFromURL = url: sep:
+        let
+          components = splitString "/" url;
+          filename = last components;
+          name = head (splitString sep filename);
+        in name;
+      defToSrc = { version, ... }@pkgdef:
+        let
+          inherit (getUrl bootstrapPackages pkgdef) src;
+          name = let n = nameFromURL pkgdef.dev-repo "."; in
+                 # rename dune so it doesn't clash with dune file in duniverse
+                 if n == "dune" then "_dune" else n;
+        in
+        # filter out pkgs without dev-repos
+        if pkgdef ? dev-repo then { inherit name version src; } else { };
+      # remove filterPkgs
+      filteredDefs = removeAttrs defs filterPkgs;
+      srcs = mapAttrsToList (pkgName: def: defToSrc def) filteredDefs;
+      # remove empty elements from pkgs without dev-repos
+      cleanedSrcs = remove { } srcs;
+    in cleanedSrcs;
+
+  deduplicateSrcs = srcs:
+    # This is O(n^2). We could try and improve this by sorting the list on name. But n is small.
+    let op = srcs: newSrc:
+      # Find if two packages come from the same dev-repo.
+      # Note we are assuming no dev-repos will have different names here, but we also assume
+      # this later when we will symlink in the duniverse directory based on this name.
+      let duplicateSrc = findSingle (src: src.name == newSrc.name) null "multiple" srcs; in
+      # Multiple duplicates should never be found as we deduplicate on every new element.
+      assert duplicateSrc != "multiple";
+      if duplicateSrc == null then srcs ++ [ newSrc ]
+      # > If packages from the same repo were resolved to different URLs, we need to pick
+      # > a single one. Here we decided to go with the one associated with the package
+      # > that has the higher version. We need a better long term solution as this won't
+      # > play nicely with pins for instance.
+      # > The best solution here would be to use source trimming, so we can pull each individual
+      # > package to its own directory and strip out all the unrelated source code but we would
+      # > need dune to provide that feature.
+      # See [opam-monorepo](https://github.com/tarides/opam-monorepo/blob/9262e7f71d749520b7e046fbd90a4732a43866e9/lib/duniverse.ml#L143-L157)
+      else if duplicateSrc.version >= newSrc.version then srcs
+      else (remove duplicateSrc srcs) ++ [ newSrc ];
+    in foldl' op [ ] srcs;
+
+  mkMonorepo = srcs:
+    let
+      # derivation that fetches the source
+      mkSrc = { name, version, src }:
+        bootstrapPackages.pkgsBuildBuild.stdenv.mkDerivation ({
+          inherit name version src;
+          phases = [ "unpackPhase" "installPhase" ];
+          installPhase = ''
+            mkdir $out
+            cp -R . $out
+          '';
+        });
+    in
+    listToAttrs (map (src: nameValuePair src.name (mkSrc src)) srcs);
+
+  queryToMonorepo = { repos ? [ mirageOpamOverlays opamOverlays opamRepository ], resolveArgs ? { }
+      , filterPkgs ? [ ] }:
+    query:
+    pipe query [
+      # pass monorepo = 1 to pick up dependencies marked with {?monorepo}
+      # TODO use opam monorepo solver to filter non-dune dependant packages
+      (opamList (joinRepos repos) (resolveArgs // { env.monorepo=1; }))
+      opamListToQuery
+      (queryToDefs repos)
+      (defsToSrcs filterPkgs)
+      deduplicateSrcs
+      mkMonorepo
+    ];
+
+  buildOpamMonorepo = { repos ? [ mirageOpamOverlays opamOverlays opamRepository ]
+    , resolveArgs ? { dev = true; }, pinDepends ? true, recursive ? false
+    , extraFilterPkgs ? [ ] }@args:
+    project: query:
+    let
+      repo = makeOpamRepo' recursive project;
+      latestVersions = mapAttrs (_: last) (listRepo repo);
+
+      pinDeps = concatLists (attrValues (mapAttrs
+        (name: version: getPinDepends repo.passthru.pkgdefs.${name}.${version})
+        latestVersions));
+    in queryToMonorepo {
+      repos = [ repo ] ++ optionals pinDepends pinDeps ++ repos;
+      filterPkgs = [ "ocaml-system" "opam-monorepo" ] ++
+        # filter all queried packages, and packages with sources
+        # in the project, from the monorepo
+        (attrNames latestVersions) ++
+        extraFilterPkgs;
+      inherit resolveArgs;
+    } (latestVersions // query);
 }