@slowtest Add support for extra sources (#1318)

* logs * install extra sources * wip: parse extra-souces and store in Dist.NoSource * write extraSources into esy.lock/index.json * wip: wire in pkg.extraSources * wip: fix paths and enable install * wip: remove un-necessary code * wip: remove dependency of Path on Empty in DistStorage * wip: cleanup code * fix: formatting * fix: e2e tests * refactor: use RunAsync.List instead of Lwt_List * run @slowtest libtorch * fix: default to [] in case of old lock files * rename fix to fmt * remove opam related dependency from esy-lib * remove un-necessary files * fmt * remove build bash script * make extraSources optional in fetch * add more information to error log * @slowtest * test for extra-source in opam dependency * @slowtest extra-source libtorch * reorder scripts & run @slowtest
esy · Jun 13, 2021 · ca4e8ef · ca4e8ef
1 parent 5ce0d22
commit ca4e8ef
Show file tree

Hide file tree

Showing 20 changed files with 242 additions and 13 deletions.
diff --git a/esy-install/DistStorage.re b/esy-install/DistStorage.re
@@ -67,15 +67,50 @@ let ofCachedTarball = path =>
   Tarball({tarballPath: path, stripComponents: 0});
 let ofDir = path => SourcePath(path);
 
-let fetch' = (sandbox, dist, gitUsername, gitPassword) => {
+let fetch' = (sandbox, dist, gitUsername, gitPassword, ~extraSources=?, ()) => {
   open RunAsync.Syntax;
   let tempPath = SandboxSpec.tempPath(sandbox);
   switch (dist) {
   | Dist.LocalPath({path: srcPath, manifest: _}) =>
     let srcPath = DistPath.toPath(sandbox.SandboxSpec.path, srcPath);
     return(SourcePath(srcPath));
 
-  | Dist.NoSource => return(Empty)
+  | Dist.NoSource =>
+    switch (extraSources) {
+    | Some(extraSources) =>
+      if (extraSources == []) {
+        return(Empty);
+      } else {
+        let path = CachePaths.fetchedDist(sandbox, dist);
+        let* () = Fs.createDir(path);
+        Fs.withTempDir(
+          ~tempPath,
+          stagePath => {
+            let%bind () = Fs.createDir(stagePath);
+            let%bind _ =
+              RunAsync.List.map(
+                ~f=
+                  ({ExtraSource.url, checksum, relativePath}) => {
+                    open RunAsync.Syntax;
+                    let tarballPath = Path.(stagePath / relativePath);
+                    let* _ = Curl.download(~output=tarballPath, url);
+                    let* _ = Checksum.checkFile(~path=tarballPath, checksum);
+                    let* _ =
+                      Fs.rename(
+                        ~skipIfExists=true,
+                        ~src=tarballPath,
+                        Path.(path / relativePath),
+                      );
+                    RunAsync.return();
+                  },
+                extraSources,
+              );
+            return(Path(path));
+          },
+        );
+      }
+    | None => return(Empty)
+    }
 
   | Dist.Archive({url, checksum}) =>
     let path = CachePaths.fetchedDist(sandbox, dist);
@@ -137,6 +172,7 @@ let fetch' = (sandbox, dist, gitUsername, gitPassword) => {
         let* () = Git.checkout(~ref=github.commit, ~repo=stagePath, ());
         let* () = Git.updateSubmodules(~config, ~repo=stagePath, ());
         let* () = Fs.rename(~skipIfExists=true, ~src=stagePath, path);
+        // TODO: handle extraSouces for Git repos
         return(Path(path));
       },
     );
@@ -166,15 +202,17 @@ let fetch' = (sandbox, dist, gitUsername, gitPassword) => {
         let* () = Git.checkout(~ref=git.commit, ~repo=stagePath, ());
         let* () = Git.updateSubmodules(~config, ~repo=stagePath, ());
         let* () = Fs.rename(~skipIfExists=true, ~src=stagePath, path);
+        // TODO: handle extraSouces for Git repos
         return(Path(path));
       },
     );
   };
 };
 
-let fetch = (_cfg, sandbox, dist, gitUsername, gitPassword) =>
+let fetch =
+    (_cfg, sandbox, dist, gitUsername, gitPassword, ~extraSources=?, ()) =>
   RunAsync.contextf(
-    fetch'(sandbox, dist, gitUsername, gitPassword),
+    fetch'(sandbox, dist, gitUsername, gitPassword, ~extraSources?, ()),
     "fetching dist: %a",
     Dist.pp,
     dist,
@@ -198,7 +236,15 @@ let unpack = (fetched, path) =>
 
       return();
     | Tarball({tarballPath, stripComponents}) =>
-      Tarball.unpack(~stripComponents, ~dst=path, tarballPath)
+      let%lwt () =
+        Logs_lwt.debug(m =>
+          m(
+            "tarballPath %s, path %s",
+            Fpath.to_string(tarballPath),
+            Fpath.to_string(path),
+          )
+        );
+      Tarball.unpack(~stripComponents, ~dst=path, tarballPath);
     }
   );
 
@@ -208,7 +254,7 @@ let fetchIntoCache = (cfg, sandbox, dist: Dist.t, gitUsername, gitPassword) => {
   if%bind (Fs.exists(path)) {
     return(path);
   } else {
-    let* fetched = fetch(cfg, sandbox, dist, gitUsername, gitPassword);
+    let* fetched = fetch(cfg, sandbox, dist, gitUsername, gitPassword, ());
     let tempPath = SandboxSpec.tempPath(sandbox);
     Fs.withTempDir(
       ~tempPath,

diff --git a/esy-install/DistStorage.rei b/esy-install/DistStorage.rei
@@ -22,7 +22,15 @@ let ofCachedTarball: Path.t => fetchedDist;
 let ofDir: Path.t => fetchedDist;
 
 let fetch:
-  (Config.t, SandboxSpec.t, Dist.t, option(string), option(string)) =>
+  (
+    Config.t,
+    SandboxSpec.t,
+    Dist.t,
+    option(string),
+    option(string),
+    ~extraSources: list(ExtraSource.t)=?,
+    unit
+  ) =>
   RunAsync.t(fetchedDist);
 
 let unpack: (fetchedDist, Path.t) => RunAsync.t(unit);

diff --git a/esy-install/Fetch.re b/esy-install/Fetch.re
@@ -259,13 +259,16 @@ module FetchPackage: {
     let rec fetchAny = (errs, alternatives) =>
       switch (alternatives) {
       | [dist, ...rest] =>
+        let extraSources = Package.extraSources(pkg);
         let fetched =
           DistStorage.fetch(
             sandbox.Sandbox.cfg,
             sandbox.spec,
             dist,
+            ~extraSources,
             gitUsername,
             gitPassword,
+            (),
           );
         switch%lwt (fetched) {
         | Ok(fetched) => return(fetched)

diff --git a/esy-install/Package.re b/esy-install/Package.re
@@ -9,6 +9,7 @@ type t = {
   dependencies: PackageId.Set.t,
   devDependencies: PackageId.Set.t,
   installConfig: InstallConfig.t,
+  extraSources: list(ExtraSource.t),
 };
 
 let compare = (a, b) => PackageId.compare(a.id, b.id);
@@ -19,6 +20,7 @@ let pp = (fmt, pkg) =>
 let show = Format.asprintf("%a", pp);
 
 let id = pkg => pkg.id;
+let extraSources = pkg => pkg.extraSources;
 
 let opam = pkg =>
   RunAsync.Syntax.(

diff --git a/esy-install/Package.rei b/esy-install/Package.rei
@@ -9,9 +9,11 @@ type t = {
   dependencies: PackageId.Set.t,
   devDependencies: PackageId.Set.t,
   installConfig: InstallConfig.t,
+  extraSources: list(ExtraSource.t),
 };
 
 let id: t => PackageId.t;
+let extraSources: t => list(ExtraSource.t);
 let opam: t => RunAsync.t(option((string, Version.t, OpamFile.OPAM.t)));
 
 include S.COMPARABLE with type t := t;

diff --git a/esy-install/SolutionLock.re b/esy-install/SolutionLock.re
@@ -50,6 +50,8 @@ and node = {
   devDependencies: PackageId.Set.t,
   [@default InstallConfig.empty]
   installConfig: InstallConfig.t,
+  [@default []]
+  extraSources: list(ExtraSource.t),
 };
 
 let indexFilename = "index.json";
@@ -228,6 +230,7 @@ let writePackage = (sandbox, pkg: Package.t, gitUsername, gitPassword) => {
     dependencies: pkg.dependencies,
     devDependencies: pkg.devDependencies,
     installConfig: pkg.installConfig,
+    extraSources: pkg.extraSources,
   });
 };
 
@@ -254,6 +257,7 @@ let readPackage = (sandbox, node: node) => {
     dependencies: node.dependencies,
     devDependencies: node.devDependencies,
     installConfig: node.installConfig,
+    extraSources: node.extraSources,
   });
 };
 

diff --git a/esy-lib/Fs.re b/esy-lib/Fs.re
@@ -224,9 +224,9 @@ let fold =
 let listDir = path =>
   switch%lwt (Lwt_unix.opendir(Path.show(path))) {
   | exception (Unix.Unix_error(Unix.ENOENT, "opendir", _)) =>
-    RunAsync.error("cannot read the directory")
+    RunAsync.errorf("cannot read the directory: %s", Fpath.to_string(path))
   | exception (Unix.Unix_error(Unix.ENOTDIR, "opendir", _)) =>
-    RunAsync.error("not a directory")
+    RunAsync.errorf("not a directory: %s", Fpath.to_string(path))
   | dir =>
     let rec readdir = (names, ()) =>
       switch%lwt (Lwt_unix.readdir(dir)) {

diff --git a/esy-package-config/ExtraSource.re b/esy-package-config/ExtraSource.re
@@ -0,0 +1,6 @@
+[@deriving (to_yojson, of_yojson({strict: false}))]
+type t = {
+  checksum: Checksum.t,
+  url: string,
+  relativePath: string,
+};
diff --git a/esy-package-config/InstallManifest.re b/esy-package-config/InstallManifest.re
@@ -149,6 +149,7 @@ type t = {
   resolutions: Resolutions.t,
   kind,
   installConfig: InstallConfig.t,
+  extraSources: list(ExtraSource.t),
 }
 and kind =
   | Esy

diff --git a/esy-package-config/InstallManifest.rei b/esy-package-config/InstallManifest.rei
@@ -42,6 +42,7 @@ type t = {
   resolutions: Resolutions.t,
   kind,
   installConfig: InstallConfig.t,
+  extraSources: list(ExtraSource.t),
 }
 and kind =
   | Esy

diff --git a/esy-package-config/OfPackageJson.re b/esy-package-config/OfPackageJson.re
@@ -182,6 +182,7 @@ module InstallManifestV1 = {
             Npm;
           },
         installConfig: pkgJson.installConfig,
+        extraSources: [],
       },
       warnings,
     ));

diff --git a/esy-solve/OpamManifest.re b/esy-solve/OpamManifest.re
@@ -237,6 +237,19 @@ let convertOpamUrl = (manifest: t) => {
   };
 };
 
+let convOpamKind = kind =>
+  switch (kind) {
+  | `MD5 => Checksum.Md5
+  | `SHA256 => Sha256
+  | `SHA512 => Sha512
+  };
+
+let opamHashToChecksum = opamHash => {
+  let kind = OpamHash.kind(opamHash) |> convOpamKind;
+  let contents = OpamHash.contents(opamHash);
+  (kind, contents);
+};
+
 let convertDependencies = manifest => {
   open Result.Syntax;
 
@@ -335,7 +348,19 @@ let convertDependencies = manifest => {
     );
   };
 
-  return((dependencies, devDependencies, optDependencies));
+  let extraSources =
+    manifest.opam
+    |> OpamFile.OPAM.extra_sources
+    |> List.map(~f=((basename, u)) => {
+         let relativePath = OpamFilename.Base.to_string(basename);
+         let url = OpamUrl.to_string(OpamFile.URL.url(u));
+         let checksum =
+           OpamFile.URL.checksum(u) |> List.hd |> opamHashToChecksum;
+
+         {ExtraSource.url, relativePath, checksum};
+       });
+
+  return((dependencies, devDependencies, optDependencies, extraSources));
 };
 
 let toInstallManifest = (~source=?, ~name, ~version, manifest) => {
@@ -344,14 +369,26 @@ let toInstallManifest = (~source=?, ~name, ~version, manifest) => {
   let converted = {
     open Result.Syntax;
     let* source = convertOpamUrl(manifest);
-    let* (dependencies, devDependencies, optDependencies) =
+    let* (dependencies, devDependencies, optDependencies, extraSources) =
       convertDependencies(manifest);
-    return((source, dependencies, devDependencies, optDependencies));
+    return((
+      source,
+      dependencies,
+      devDependencies,
+      optDependencies,
+      extraSources,
+    ));
   };
 
   switch (converted) {
   | Error(err) => return(Error(err))
-  | Ok((sourceFromOpam, dependencies, devDependencies, optDependencies)) =>
+  | Ok((
+      sourceFromOpam,
+      dependencies,
+      devDependencies,
+      optDependencies,
+      extraSources,
+    )) =>
     let opam =
       switch (manifest.opamRepositoryPath) {
       | Some(path) =>
@@ -388,6 +425,7 @@ let toInstallManifest = (~source=?, ~name, ~version, manifest) => {
         peerDependencies: NpmFormula.empty,
         resolutions: Resolutions.empty,
         installConfig: InstallConfig.empty,
+        extraSources,
       }),
     );
   };

diff --git a/esy-solve/Resolver.re b/esy-solve/Resolver.re
@@ -95,6 +95,7 @@ let emptyLink = (~name, ~path, ~manifest, ~kind, ()) => {
   resolutions: Resolutions.empty,
   kind: Esy,
   installConfig: InstallConfig.empty,
+  extraSources: [],
 };
 
 let emptyInstall = (~name, ~source, ()) => {
@@ -111,6 +112,7 @@ let emptyInstall = (~name, ~source, ()) => {
   resolutions: Resolutions.empty,
   kind: Esy,
   installConfig: InstallConfig.empty,
+  extraSources: [],
 };
 
 let make = (~cfg, ~sandbox, ()) =>

diff --git a/esy-solve/Sandbox.re b/esy-solve/Sandbox.re
@@ -125,6 +125,7 @@ let make = (~gitUsername, ~gitPassword, ~cfg, spec: EsyInstall.SandboxSpec.t) =>
           resolutions,
           kind: Npm,
           installConfig: InstallConfig.empty,
+          extraSources: [],
         };
         return({cfg, spec, root, resolutions: root.resolutions, resolver});
       };

diff --git a/esy-solve/Solver.re b/esy-solve/Solver.re
@@ -391,6 +391,7 @@ let lockPackage =
     resolutions: _,
     kind: _,
     installConfig,
+    extraSources,
   } = pkg;
 
   let idsOfDependencies = dependencies =>
@@ -470,6 +471,7 @@ let lockPackage =
     dependencies,
     devDependencies,
     installConfig,
+    extraSources,
   });
 };
 
@@ -654,6 +656,7 @@ let solveDependencies =
     resolutions: Resolutions.empty,
     kind: Esy,
     installConfig: InstallConfig.empty,
+    extraSources: [],
   };
 
   let universe = Universe.add(~pkg=dummyRoot, solver.universe);

diff --git a/esy.json b/esy.json
@@ -26,6 +26,7 @@
     "build:postinstall-esy-bash": "cd node_modules/esy-bash && npm install",
     "build:postinstall-esy-solve-cudf": "cd node_modules/esy-solve-cudf && npm install",
     "release:platform-release": "node ./scripts/make-platform-release.js",
+    "fmt": "dune build @fmt --auto-promote",
     "test:unit": "esy b dune runtest",
     "test:e2e": "npm run jest test-e2e",
     "test:e2e-re": "sh -c 'esy b; esy dune exec ./test-e2e-re/lib/RunTests.exe'",

diff --git a/test-e2e-slow/build-top-100-opam.test.js b/test-e2e-slow/build-top-100-opam.test.js
@@ -124,6 +124,7 @@ let cases = [
    // TODO { name: "ppx_has", toolchains: [ocamlVersion] },
    // Blocked by esy/esy#505
    // {name: 'coq', toolchains: [ocamlVersion]},
+   { name: 'libtorch', toolchains: [ocamlVersion] },
 ];
 
 let reposUpdated = false;