Towards #102: Out-of-tree source management

crate2nix.sh

@@ -8,4 +8,4 @@ set -Eeuo pipefail
 
 mydir=$(dirname "$0")
 
-nix run -f "${mydir}" -c crate2nix "$@"
+nix run -f "${mydir}" --arg release "${RELEASE:-true}" -c crate2nix "$@"

crate2nix/Cargo.nix

@@ -16,6 +16,11 @@
 , strictDeprecation ? false
   # Whether to perform release builds: longer compile times, faster binaries.
 , release ? true
+  # Additional crate2nix configuration if it exists.
+, crateConfig
+  ? if builtins.pathExists ./crate-config.nix
+    then pkgs.callPackage ./crate-config.nix {}
+    else {}
 }:
 
 rec {
@@ -2819,19 +2824,20 @@ rec {
               };
               drv = builtRustCrates.${packageId};
               testDrv = builtTestRustCrates.${packageId};
-            in
-              if runTests then
+              derivation = if runTests then
                 crateWithTest {
                   crate = drv;
                   testCrate = testDrv;
                   inherit testCrateFlags testInputs;
                 }
-              else drv
+              else drv;
+            in
+              derivation
         )
         { inherit features crateOverrides runTests testCrateFlags testInputs; };
 
-  /* Returns an attr set with packageId mapped to the result of buildRustCrateFunc 
-     for the corresponding crate. 
+  /* Returns an attr set with packageId mapped to the result of buildRustCrateFunc
+     for the corresponding crate.
   */
   builtRustCratesWithFeatures =
     { packageId
@@ -2937,7 +2943,7 @@ rec {
         map depDerivation enabledDependencies;
 
   /* Returns a sanitized version of val with all values substituted that cannot
-     be serialized as JSON. 
+     be serialized as JSON.
   */
   sanitizeForJson = val:
     if builtins.isAttrs val
@@ -2982,7 +2988,7 @@ rec {
 
   /* Returns differences between cargo default features and crate2nix default
      features.
-   
+
      This is useful for verifying the feature resolution in crate2nix.
   */
   diffDefaultPackageFeatures =
@@ -3155,7 +3161,7 @@ rec {
 
   /*
      Returns the actual features for the given dependency.
-    
+
      features: The features of the crate that refers this dependency.
   */
   dependencyFeatures = features: dependency:

crate2nix/src/command.rs

@@ -0,0 +1,74 @@
+//! Utilities for dealing with spawned commands.
+
+use failure::{bail, format_err, Error};
+use std::process::{Child, Stdio};
+use std::thread;
+use std::{
+    io::{BufRead, Read},
+    io::{BufReader, Cursor},
+    sync::mpsc,
+};
+
+/// Runs the given command with output capturing.
+///
+/// The output will be printed indented if and only if the command does not
+/// return succesfully.
+pub fn run(caption: &str, command: &mut std::process::Command) -> Result<(), Error> {
+    eprint!("{}: ", caption);
+
+    let mut spawned: Child = command
+        .stdin(Stdio::null())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|e| format_err!("while spawning {:?}: {}", command, e))?;
+
+    let (sender, receiver) = mpsc::channel();
+
+    pass_through(spawned.stdout.take().expect("stdout"), sender.clone());
+    pass_through(spawned.stderr.take().expect("stderr"), sender);
+
+    let mut out = Vec::<u8>::new();
+    while let Ok(buf) = receiver.recv() {
+        out.extend(buf.iter());
+    }
+
+    let status = spawned
+        .wait()
+        .map_err(|e| format_err!("while waiting for the {:?} to finish: {}", command, e))?;
+
+    if status.success() {
+        eprintln!("done.");
+        return Ok(());
+    }
+
+    eprintln!();
+    eprintln!("  {:?}", command);
+    let line_reader = BufReader::new(Cursor::new(out));
+    for line in line_reader.lines() {
+        println!(
+            "  {}",
+            line.map_err(|e| format_err!("while processing output lines: {}", e))?
+        );
+    }
+
+    bail!(
+        "{:?}\n=> exited with: {}",
+        command,
+        status.code().unwrap_or(-1)
+    );
+}
+
+fn pass_through(mut read: impl Read + Send + 'static, sender: mpsc::Sender<Vec<u8>>) {
+    thread::spawn(move || {
+        let mut buf = [0; 4096];
+        while let Ok(n) = read.read(&mut buf) {
+            if n == 0 {
+                break;
+            }
+            if sender.send(Vec::from(&buf[..n])).is_err() {
+                break;
+            }
+        }
+    });
+}

crate2nix/src/config.rs

@@ -0,0 +1,243 @@
+//! Managing the `crate2nix.json` config.
+
+use failure::{bail, Error, ResultExt};
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::BTreeMap,
+    fmt::Display,
+    fs::File,
+    io::{BufReader, BufWriter},
+    path::Path,
+    str::FromStr,
+};
+
+impl Config {
+    /// Read config from path.
+    pub fn read_from_or_default(path: &Path) -> Result<Config, Error> {
+        if !path.exists() {
+            return Ok(Config::default());
+        }
+
+        let file = File::open(path).context(format!("while opening {}", path.to_string_lossy()))?;
+        let reader = BufReader::new(file);
+        Ok(serde_json::from_reader(reader).context(format!(
+            "while deserializing config: {}",
+            path.to_string_lossy()
+        ))?)
+    }
+
+    /// Write config to path.
+    pub fn write_to(&self, path: &Path) -> Result<(), Error> {
+        let file =
+            File::create(path).context(format!("while opening {}", path.to_string_lossy()))?;
+        let writer = BufWriter::new(file);
+        Ok(serde_json::to_writer_pretty(writer, self)?)
+    }
+}
+
+/// The `crate2nix.json` config data.
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct Config {
+    /// Out of tree sources.
+    pub sources: BTreeMap<String, Source>,
+}
+
+impl Config {
+    /// Add or replace a source. Returns the old source if there was one.
+    pub fn upsert_source(
+        &mut self,
+        explicit_name: Option<String>,
+        source: Source,
+    ) -> Option<Source> {
+        let name = source
+            .name()
+            .map(|s| s.to_string())
+            .or(explicit_name)
+            .expect("No name given");
+        self.sources.insert(name, source)
+    }
+
+    /// Prints all sources to stdout.
+    pub fn print_sources(&self) {
+        if self.sources.is_empty() {
+            eprintln!("No sources configured.\n");
+            return;
+        }
+
+        let max_len = self
+            .sources
+            .iter()
+            .map(|(n, _)| n.len())
+            .max()
+            .unwrap_or_default();
+        for (name, source) in &self.sources {
+            println!("{:width$} {}", name, source, width = max_len);
+            println!();
+            println!(
+                "{:width$} crate2nix source add {}",
+                "",
+                source.as_command(name),
+                width = max_len
+            );
+            println!();
+        }
+    }
+}
+
+/// An out of tree source.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(tag = "type")]
+pub enum Source {
+    /// Get the source from crates.io.
+    CratesIo {
+        /// The crate name.
+        name: String,
+        /// The exact crate version to fetch.
+        version: semver::Version,
+        /// The sha256 hash of the source.
+        sha256: String,
+    },
+    /// Get the source from git.
+    Git {
+        /// The URL of the git repository.
+        ///
+        /// E.g. https://github.com/kolloch/crate2nix.git
+        #[serde(with = "url_serde")]
+        url: url::Url,
+        /// The revision hash.
+        rev: String,
+        /// The sha256 of the fetched result.
+        sha256: String,
+    },
+    /// Get the source from a nix expression.
+    Nix {
+        /// The nixfile to include.
+        #[serde(flatten)]
+        file: NixFile,
+        /// A Nix attribute path which will be resolved against the file.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        attr: Option<String>,
+    },
+}
+
+/// A nix file path which is either included by `import` or `callPackage`.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq, Hash)]
+pub enum NixFile {
+    /// A file path that should be imported.
+    #[serde(rename = "import")]
+    Import(String),
+    /// A file path the should be included by `pkgs.callPackage`.
+    #[serde(rename = "package")]
+    Package(String),
+}
+
+impl Display for NixFile {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Import(path) => write!(f, "import {}", path),
+            Self::Package(path) => write!(f, "pkgs.callPackage {} {{}}", path),
+        }
+    }
+}
+
+impl NixFile {
+    /// Returns the chosen file option as CLI string.
+    pub fn as_command(&self) -> String {
+        match self {
+            Self::Import(path) => format!("--import '{}'", path),
+            Self::Package(path) => format!("--package '{}'", path),
+        }
+    }
+}
+
+impl Source {
+    /// The name of the source.
+    pub fn name(&self) -> Option<&str> {
+        match self {
+            Source::CratesIo { name, .. } => Some(name),
+            Source::Git { url, .. } => {
+                let path = url.path();
+                let after_last_slash = path.split('/').last().unwrap_or_else(|| path);
+                let without_dot_git = strip_suffix(after_last_slash, ".git");
+                Some(without_dot_git)
+            }
+            Source::Nix {
+                attr: Some(attr), ..
+            } => attr.split('.').last().or(if attr.trim().is_empty() {
+                None
+            } else {
+                Some(attr.trim())
+            }),
+            _ => None,
+        }
+    }
+}
+
+fn strip_suffix<'a>(s: &'a str, p: &str) -> &'a str {
+    if s.ends_with(p) {
+        &s[..s.len() - p.len()]
+    } else {
+        s
+    }
+}
+
+impl Display for Source {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Source::CratesIo {
+                name,
+                version,
+                sha256,
+            } => write!(f, "{} {} from crates.io: {}", name, version, sha256),
+            Source::Git { url, rev, sha256 } => write!(f, "{}#{} via git: {}", url, rev, sha256),
+            Source::Nix { file, attr: None } => write!(f, "{}", file),
+            Source::Nix {
+                file,
+                attr: Some(attr),
+            } => write!(f, "({}).{}", file, attr),
+        }
+    }
+}
+
+impl Source {
+    /// Returns a CLI string to reproduce this source.
+    pub fn as_command(&self, name: &str) -> String {
+        match self {
+            Source::CratesIo {
+                name: crate_name,
+                version,
+                ..
+            } => format!("cratesIo --name '{}' '{}' '{}'", name, crate_name, version),
+            Source::Git { url, rev, .. } => {
+                format!("git --name '{}' '{}' --rev {}", name, url, rev)
+            }
+            Source::Nix { file, attr: None } => {
+                format!("nix --name '{}' {}", name, file.as_command())
+            }
+            Source::Nix {
+                file,
+                attr: Some(attr),
+            } => format!("nix --name '{}' {} '{}'", name, file.as_command(), attr),
+        }
+    }
+}
+
+/// The type of a Source.
+#[derive(Debug, Serialize, Deserialize)]
+pub enum SourceType {
+    /// Corresponds to Source::CratesIo.
+    CratesIo,
+    /// Corresponds to Source::Git.
+    Git,
+}
+
+impl FromStr for SourceType {
+    type Err = failure::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "cratesIo" => Ok(SourceType::CratesIo),
+            "git" => Ok(SourceType::Git),
+            _ => bail!("unkown source type: {}", s),
+        }
+    }
+}