Add support for submodules and release artifacts

.envrc

@@ -1 +1,2 @@
+watch_file npins/*
 use_nix

.github/workflows/test.yml

@@ -16,6 +16,10 @@ jobs:
         run: nix-build
       - name: Run pre-commit hooks
         run: nix-shell --run "pre-commit run --all"
+      - name: Test dev shell
+        # Simple check that the pins in the current repository are still working.
+        # Importantly, this will fail on any version mismatch, indicating that versions need to be upgraded.
+        run: nix-shell --run "npins show"
       - name: Run smoke test
         run: nix-shell --run "bash ./.github/workflows/smoke-test.sh"
       - name: Run integration tests

README.md

@@ -223,7 +223,8 @@ FLAGS:
 OPTIONS:
     -d, --directory <folder>    Base folder for sources.json and the boilerplate default.nix [env: NPINS_DIRECTORY=]
                                 [default: npins]
-        --name <name>           Custom name for the pin entry
+        --name <name>           Add the pin with a custom name. If a pin with that name already exists, it willl be
+                                overwritten
 
 SUBCOMMANDS:
     channel    Track a Nix channel
@@ -247,6 +248,7 @@ USAGE:
 FLAGS:
     -h, --help            Prints help information
         --pre-releases    Also track pre-releases. Conflicts with the --branch option
+        --submodules      Also fetch submodules
 
 OPTIONS:
         --at <tag or rev>                    Use a specific commit/release instead of the latest. This may be a tag

npins.nix

@@ -16,9 +16,7 @@
 let
   paths = [
     "^/src$"
-    "^/src/.+.rs$"
-    "^/npins$"
-    "^/npins/default.nix$"
+    "^/src/.+$"
     "^/Cargo.lock$"
     "^/Cargo.toml$"
   ];

shell.nix

@@ -50,6 +50,7 @@ pkgs.mkShell {
       nix_2_3
       nix-prefetch-git
       git
+      npins
     ]
     ++ (lib.optionals stdenv.isDarwin [
       pkgs.libiconv

src/cli.rs

@@ -9,6 +9,8 @@ use structopt::StructOpt;
 
 use url::Url;
 
+const DEFAULT_NIX: &'static str = include_str!("default.nix");
+
 /// How to handle updates
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 pub enum UpdateStrategy {
@@ -87,6 +89,10 @@ pub struct GenericGitAddOpts {
     /// that start with that string.
     #[structopt(long = "release-prefix")]
     pub release_prefix: Option<String>,
+
+    /// Also fetch submodules
+    #[structopt(long)]
+    pub submodules: bool,
 }
 
 #[derive(Debug, StructOpt)]
@@ -104,7 +110,12 @@ impl GitHubAddOpts {
             self.repository.clone(),
             match &self.more.branch {
                 Some(branch) => {
-                    let pin = git::GitPin::github(&self.owner, &self.repository, branch.clone());
+                    let pin = git::GitPin::github(
+                        &self.owner,
+                        &self.repository,
+                        branch.clone(),
+                        self.more.submodules,
+                    );
                     let version = self.more.at.as_ref().map(|at| git::GitRevision {
                         revision: at.clone(),
                     });
@@ -117,6 +128,7 @@ impl GitHubAddOpts {
                         self.more.pre_releases,
                         self.more.version_upper_bound.clone(),
                         self.more.release_prefix.clone(),
+                        self.more.submodules,
                     );
                     let version = self.more.at.as_ref().map(|at| GenericVersion {
                         version: at.clone(),
@@ -144,7 +156,7 @@ pub struct GitLabAddOpts {
 
     #[structopt(
         long,
-        help = "Use a private token to access the repository",
+        help = "Use a private token to access the repository.",
         value_name = "token"
     )]
     pub private_token: Option<String>,
@@ -167,6 +179,7 @@ impl GitLabAddOpts {
                         branch.clone(),
                         Some(self.server.clone()),
                         self.private_token.clone(),
+                        self.more.submodules,
                     );
                     let version = self.more.at.as_ref()
                     .map(|at| git::GitRevision {
@@ -180,7 +193,8 @@ impl GitLabAddOpts {
                         self.more.pre_releases,
                         self.more.version_upper_bound.clone(),
                         self.private_token.clone(),
-			self.more.release_prefix.clone(),
+                        self.more.release_prefix.clone(),
+                        self.more.submodules,
                     );
                     let version = self.more.at.as_ref()
                         .map(|at| GenericVersion {
@@ -232,7 +246,7 @@ impl GitAddOpts {
             name.to_owned(),
             match &self.more.branch {
                 Some(branch) => {
-                    let pin = git::GitPin::git(url, branch.clone());
+                    let pin = git::GitPin::git(url, branch.clone(), self.more.submodules);
                     let version = self.more.at.as_ref().map(|at| git::GitRevision {
                         revision: at.clone(),
                     });
@@ -244,6 +258,7 @@ impl GitAddOpts {
                         self.more.pre_releases,
                         self.more.version_upper_bound.clone(),
                         self.more.release_prefix.clone(),
+                        self.more.submodules,
                     );
                     let version = self.more.at.as_ref().map(|at| GenericVersion {
                         version: at.clone(),
@@ -306,7 +321,8 @@ pub enum AddCommands {
 
 #[derive(Debug, StructOpt)]
 pub struct AddOpts {
-    /// Custom name for the pin entry
+    /// Add the pin with a custom name.
+    /// If a pin with that name already exists, it willl be overwritten
     #[structopt(long)]
     pub name: Option<String>,
     /// Don't actually apply the changes
@@ -478,15 +494,15 @@ impl Opts {
 
     async fn init(&self, o: &InitOpts) -> Result<()> {
         log::info!("Welcome to npins!");
-        let default_nix = include_bytes!("../npins/default.nix");
+        let default_nix = DEFAULT_NIX;
         if !self.folder.exists() {
             log::info!("Creating `{}` directory", self.folder.display());
             std::fs::create_dir(&self.folder).context("Failed to create npins folder")?;
         }
         log::info!("Writing default.nix");
         let p = self.folder.join("default.nix");
         let mut fh = std::fs::File::create(&p).context("Failed to create npins default.nix")?;
-        fh.write_all(default_nix)?;
+        fh.write_all(default_nix.as_bytes())?;
 
         // Only create the pins if the file isn't there yet
         if self.folder.join("sources.json").exists() {
@@ -614,7 +630,7 @@ impl Opts {
         );
 
         let nix_path = self.folder.join("default.nix");
-        let nix_file = include_str!("../npins/default.nix");
+        let nix_file = DEFAULT_NIX;
         if std::fs::read_to_string(&nix_path)? == nix_file {
             log::info!("default.nix is already up to date");
         } else {

src/default.nix

@@ -0,0 +1,89 @@
+# Generated by npins. Do not modify; will be overwritten regularly
+let
+  data = builtins.fromJSON (builtins.readFile ./sources.json);
+  version = data.version;
+
+  mkSource =
+    spec:
+    assert spec ? type;
+    let
+      path =
+        if spec.type == "Git" then
+          mkGitSource spec
+        else if spec.type == "GitRelease" then
+          mkGitSource spec
+        else if spec.type == "PyPi" then
+          mkPyPiSource spec
+        else if spec.type == "Channel" then
+          mkChannelSource spec
+        else
+          builtins.throw "Unknown source type ${spec.type}";
+    in
+    spec // { outPath = path; };
+
+  mkGitSource =
+    {
+      repository,
+      revision,
+      url ? null,
+      submodules,
+      hash,
+      branch ? null,
+      ...
+    }:
+    assert repository ? type;
+    # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository
+    # In the latter case, there we will always be an url to the tarball
+    if url != null && !submodules then
+      builtins.fetchTarball {
+        inherit url;
+        sha256 = hash; # FIXME: check nix version & use SRI hashes
+      }
+    else
+      let
+        url =
+          if repository.type == "Git" then
+            repository.url
+          else if repository.type == "GitHub" then
+            "https://github.com/${repository.owner}/${repository.repo}.git"
+          else if repository.type == "GitLab" then
+            "${repository.server}/${repository.repo_path}.git"
+          else
+            throw "Unrecognized repository type ${repository.type}";
+        urlToName =
+          url: rev:
+          let
+            matched = builtins.match "^.*/([^/]*)(\\.git)?$" repository.url;
+
+            short = builtins.substring 0 7 rev;
+
+            appendShort = if (builtins.match "[a-f0-9]*" rev) != null then "-${short}" else "";
+          in
+          "${if matched == null then "source" else builtins.head matched}${appendShort}";
+        name = urlToName url revision;
+      in
+      builtins.fetchGit {
+        rev = revision;
+        inherit name;
+        # hash = hash;
+        inherit url submodules;
+      };
+
+  mkPyPiSource =
+    { url, hash, ... }:
+    builtins.fetchurl {
+      inherit url;
+      sha256 = hash;
+    };
+
+  mkChannelSource =
+    { url, hash, ... }:
+    builtins.fetchTarball {
+      inherit url;
+      sha256 = hash;
+    };
+in
+if version == 4 then
+  builtins.mapAttrs (_: mkSource) data.pins
+else
+  throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`"

src/flake.rs

@@ -88,6 +88,7 @@ impl TryFrom<FlakePin> for Pin {
                 flake.original.ref_.unwrap_or_else(|| "master".to_owned()),
                 None,
                 None,
+                false,
             )
             .into(),
             Github => git::GitPin::github(
@@ -100,6 +101,7 @@ impl TryFrom<FlakePin> for Pin {
                     .repo
                     .context("missing field repo in github flake input")?,
                 flake.original.ref_.unwrap_or_else(|| "master".to_owned()),
+                false,
             )
             .into(),
             Git => {
@@ -113,6 +115,7 @@ impl TryFrom<FlakePin> for Pin {
                 git::GitPin::git(
                     flake.locked.url.context("missing url on git flake input")?,
                     ref_,
+                    false,
                 )
                 .into()
             },