From f27448bb5a4c0f29a5cd092215df0dfc57498388 Mon Sep 17 00:00:00 2001 From: Daniel Eades Date: Sat, 21 Mar 2020 09:14:29 +0000 Subject: [PATCH 1/5] skip serialising empty crate metadata values --- src/api/publish.rs | 2 +- src/index/models.rs | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/api/publish.rs b/src/api/publish.rs index 79fe5846..b03b9a19 100644 --- a/src/api/publish.rs +++ b/src/api/publish.rs @@ -221,7 +221,7 @@ pub(crate) async fn put(mut req: Request) -> Result { optional: dep.optional, default_features: dep.default_features, target: dep.target, - kind: dep.kind, + kind: dep.kind.unwrap_or(CrateDependencyKind::Normal), registry: dep.registry, package, } diff --git a/src/index/models.rs b/src/index/models.rs index 9e111b04..257b9950 100644 --- a/src/index/models.rs +++ b/src/index/models.rs @@ -11,17 +11,26 @@ use serde::{Deserialize, Serialize}; pub struct CrateVersion { /// The name of the crate. pub name: String, + /// The version of the crate. pub vers: Version, + /// The dependencies of the crate. + #[serde(skip_serializing_if = "Vec::is_empty", default)] pub deps: Vec, + /// The SHA256 hash of the crate. pub cksum: String, + /// The available features of the crates and what they enable. + #[serde(skip_serializing_if = "HashMap::is_empty", default)] pub features: HashMap>, + /// Is the crate yanked. pub yanked: Option, + /// Is the crate yanked. + #[serde(skip_serializing_if = "Option::is_none", default)] pub links: Option, } @@ -33,27 +42,39 @@ pub struct CrateDependency { /// If the dependency is renamed, this is the new name. /// The original name is specified in the `package` field. pub name: String, + /// The version requirement for the dependency (eg. "^1.2.0"). pub req: VersionReq, + /// The features requested for the dependency. + #[serde(skip_serializing_if = "Vec::is_empty", default)] pub features: Vec, + /// Is the dependency optional. pub optional: bool, + /// Whether the crates uses the default features of this dependency. pub default_features: bool, + /// The target platform of the dependency. /// /// A string such as "cfg(windows)" + #[serde(skip_serializing_if = "Option::is_none", default)] pub target: Option, + /// The kind of the dependency ("normal", "build" or "dev"). - pub kind: Option, + pub kind: CrateDependencyKind, + /// The URL of the index of the registry where this dependency is from. /// /// If not specified, it is assumed to come from the current registry. + #[serde(skip_serializing_if = "Option::is_none", default)] pub registry: Option, + /// If the dependency is renamed, this is the actual original crate name. /// /// If not specified, the dependency has not been renamed. + #[serde(skip_serializing_if = "Option::is_none", default)] pub package: Option, } From e0e2d2541fd642043d30186c925e1f9e426afecf Mon Sep 17 00:00:00 2001 From: Daniel Eades Date: Sat, 21 Mar 2020 09:17:31 +0000 Subject: [PATCH 2/5] deglob import --- src/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index 7d1ead99..84a98eeb 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -9,7 +9,7 @@ mod models; #[cfg(feature = "git2")] pub mod git2; -pub use models::*; +pub use models::{CrateDependency, CrateDependencyKind, CrateVersion}; use crate::error::Error; use crate::index::cli::CommandLineIndex; From cda9d20a58bb245356f0a16e46b1dde38681e692 Mon Sep 17 00:00:00 2001 From: Daniel Eades Date: Mon, 23 Mar 2020 19:53:06 +0000 Subject: [PATCH 3/5] refactor the 'cli' index split the concerns of managing the filesystem and managing the git repository into separate objects --- src/config/index/cli.rs | 2 +- src/index/cli.rs | 153 ++++++++++++++++++++++++++-------------- 2 files changed, 103 insertions(+), 52 deletions(-) diff --git a/src/config/index/cli.rs b/src/config/index/cli.rs index 4a23e11d..5c5d3b06 100644 --- a/src/config/index/cli.rs +++ b/src/config/index/cli.rs @@ -19,6 +19,6 @@ pub struct CommandLineIndexConfig { impl From for CommandLineIndex { fn from(config: CommandLineIndexConfig) -> CommandLineIndex { - CommandLineIndex { path: config.path } + CommandLineIndex::new(config.path) } } diff --git a/src/index/cli.rs b/src/index/cli.rs index ce948329..25341dd9 100644 --- a/src/index/cli.rs +++ b/src/index/cli.rs @@ -13,74 +13,70 @@ use crate::index::{CrateVersion, Indexer}; /// It manages the crate index through the invocation of "git" shell commands. #[derive(Debug, Clone, PartialEq)] pub struct CommandLineIndex { - /// The path of the crate index. - pub(crate) path: PathBuf, + repo: Repository, + tree: Tree, } impl CommandLineIndex { /// Create a CommandLineIndex instance with the given path. - pub fn new>(path: P) -> Result { + pub fn new>(path: P) -> CommandLineIndex { let path = path.into(); - Ok(CommandLineIndex { path }) - } - - fn compute_record_path(&self, name: &str) -> PathBuf { - match name.len() { - 1 => self.path.join("1").join(&name), - 2 => self.path.join("2").join(&name), - 3 => self.path.join("3").join(&name[..1]).join(&name), - _ => self.path.join(&name[0..2]).join(&name[2..4]).join(&name), - } + let repo = Repository { path: path.clone() }; + let tree = Tree { path }; + CommandLineIndex { repo, tree } } } impl Indexer for CommandLineIndex { fn url(&self) -> Result { - let output = Command::new("git") - .arg("remote") - .arg("get-url") - .arg("origin") - .stdout(Stdio::piped()) - .current_dir(self.path.canonicalize()?) - .output()?; - - Ok(String::from_utf8_lossy(output.stdout.as_slice()).into()) + self.repo.url() } fn refresh(&self) -> Result<(), Error> { - Command::new("git") - .arg("pull") - .arg("--ff-only") - .current_dir(self.path.canonicalize()?) - .spawn()? - .wait()?; - - Ok(()) + self.repo.refresh() } fn commit_and_push(&self, msg: &str) -> Result<(), Error> { - Command::new("git") - .arg("add") - .arg("--all") - .current_dir(&self.path) - .spawn()? - .wait()?; - Command::new("git") - .arg("commit") - .arg("-m") - .arg(msg) - .current_dir(&self.path) - .spawn()? - .wait()?; - Command::new("git") - .arg("push") - .arg("origin") - .arg("master") - .current_dir(&self.path) - .spawn()? - .wait()?; + self.repo.commit_and_push(msg) + } - Ok(()) + fn match_record(&self, name: &str, req: VersionReq) -> Result { + self.tree.match_record(name, req) + } + + fn all_records(&self, name: &str) -> Result, Error> { + self.tree.all_records(name) + } + + fn latest_record(&self, name: &str) -> Result { + self.tree.latest_record(name) + } + + fn add_record(&self, record: CrateVersion) -> Result<(), Error> { + self.tree.add_record(record) + } + + fn alter_record(&self, name: &str, version: Version, func: F) -> Result<(), Error> + where + F: FnOnce(&mut CrateVersion), + { + self.tree.alter_record(name, version, func) + } +} + +#[derive(Debug, Clone, PartialEq)] +struct Tree { + path: PathBuf, +} + +impl Tree { + fn compute_record_path(&self, name: &str) -> PathBuf { + match name.len() { + 1 => self.path.join("1").join(&name), + 2 => self.path.join("2").join(&name), + 3 => self.path.join("3").join(&name[..1]).join(&name), + _ => self.path.join(&name[0..2]).join(&name[2..4]).join(&name), + } } fn match_record(&self, name: &str, req: VersionReq) -> Result { @@ -195,3 +191,58 @@ impl Indexer for CommandLineIndex { Ok(()) } } + +#[derive(Debug, Clone, PartialEq)] +struct Repository { + path: PathBuf, +} + +impl Repository { + fn url(&self) -> Result { + let output = Command::new("git") + .arg("remote") + .arg("get-url") + .arg("origin") + .stdout(Stdio::piped()) + .current_dir(self.path.canonicalize()?) + .output()?; + + Ok(String::from_utf8_lossy(output.stdout.as_slice()).into()) + } + + fn refresh(&self) -> Result<(), Error> { + Command::new("git") + .arg("pull") + .arg("--ff-only") + .current_dir(self.path.canonicalize()?) + .spawn()? + .wait()?; + + Ok(()) + } + + fn commit_and_push(&self, msg: &str) -> Result<(), Error> { + Command::new("git") + .arg("add") + .arg("--all") + .current_dir(&self.path) + .spawn()? + .wait()?; + Command::new("git") + .arg("commit") + .arg("-m") + .arg(msg) + .current_dir(&self.path) + .spawn()? + .wait()?; + Command::new("git") + .arg("push") + .arg("origin") + .arg("master") + .current_dir(&self.path) + .spawn()? + .wait()?; + + Ok(()) + } +} From 56b339b88f701130ac303dceb7f137b3a16c4475 Mon Sep 17 00:00:00 2001 From: Daniel Eades Date: Mon, 23 Mar 2020 20:04:07 +0000 Subject: [PATCH 4/5] move the index::Tree into a separate module --- src/index/cli.rs | 135 +------------------------------------------- src/index/mod.rs | 1 + src/index/tree.rs | 141 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 132 deletions(-) create mode 100644 src/index/tree.rs diff --git a/src/index/cli.rs b/src/index/cli.rs index 25341dd9..f8149d0c 100644 --- a/src/index/cli.rs +++ b/src/index/cli.rs @@ -1,11 +1,10 @@ -use std::fs; -use std::io::{self, BufRead, Write}; use std::path::PathBuf; use std::process::{Command, Stdio}; use semver::{Version, VersionReq}; -use crate::error::{AlexError, Error}; +use crate::error::Error; +use crate::index::tree::Tree; use crate::index::{CrateVersion, Indexer}; /// The 'command-line' crate index management strategy type. @@ -22,7 +21,7 @@ impl CommandLineIndex { pub fn new>(path: P) -> CommandLineIndex { let path = path.into(); let repo = Repository { path: path.clone() }; - let tree = Tree { path }; + let tree = Tree::new(path); CommandLineIndex { repo, tree } } } @@ -64,134 +63,6 @@ impl Indexer for CommandLineIndex { } } -#[derive(Debug, Clone, PartialEq)] -struct Tree { - path: PathBuf, -} - -impl Tree { - fn compute_record_path(&self, name: &str) -> PathBuf { - match name.len() { - 1 => self.path.join("1").join(&name), - 2 => self.path.join("2").join(&name), - 3 => self.path.join("3").join(&name[..1]).join(&name), - _ => self.path.join(&name[0..2]).join(&name[2..4]).join(&name), - } - } - - fn match_record(&self, name: &str, req: VersionReq) -> Result { - let path = self.compute_record_path(name); - let file = fs::File::open(path).map_err(|err| match err.kind() { - io::ErrorKind::NotFound => Error::from(AlexError::CrateNotFound { - name: String::from(name), - }), - _ => Error::from(err), - })?; - let found = io::BufReader::new(file) - .lines() - .map(|line| Some(json::from_str(line.ok()?.as_str()).ok()?)) - .flat_map(|ret: Option| ret.into_iter()) - .filter(|krate| req.matches(&krate.vers)) - .max_by(|k1, k2| k1.vers.cmp(&k2.vers)); - Ok(found.ok_or_else(|| AlexError::CrateNotFound { - name: String::from(name), - })?) - } - - fn all_records(&self, name: &str) -> Result, Error> { - let path = self.compute_record_path(name); - let reader = io::BufReader::new(fs::File::open(path)?); - reader - .lines() - .map(|line| Ok(json::from_str::(line?.as_str())?)) - .collect() - } - - fn latest_record(&self, name: &str) -> Result { - let records = self.all_records(name)?; - Ok(records - .into_iter() - .max_by(|k1, k2| k1.vers.cmp(&k2.vers)) - .expect("at least one version should exist")) - } - - fn add_record(&self, record: CrateVersion) -> Result<(), Error> { - let path = self.compute_record_path(record.name.as_str()); - - if let Ok(file) = fs::File::open(&path) { - let reader = io::BufReader::new(file); - let records = reader - .lines() - .map(|line| Ok(json::from_str::(line?.as_str())?)) - .collect::, Error>>()?; - let latest = records - .into_iter() - .max_by(|k1, k2| k1.vers.cmp(&k2.vers)) - .expect("at least one record should exist"); - if record.vers <= latest.vers { - return Err(Error::from(AlexError::VersionTooLow { - krate: record.name, - hosted: latest.vers, - published: record.vers, - })); - } - } else { - let parent = path.parent().unwrap(); - fs::create_dir_all(parent)?; - } - - let mut file = fs::OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(path)?; - json::to_writer(&mut file, &record)?; - writeln!(file)?; - file.flush()?; - - Ok(()) - } - - fn alter_record(&self, name: &str, version: Version, func: F) -> Result<(), Error> - where - F: FnOnce(&mut CrateVersion), - { - let path = self.compute_record_path(name); - let file = fs::File::open(path.as_path()).map_err(|err| match err.kind() { - io::ErrorKind::NotFound => Error::from(AlexError::CrateNotFound { - name: String::from(name), - }), - _ => Error::from(err), - })?; - let mut krates: Vec = { - let mut out = Vec::new(); - for line in io::BufReader::new(file).lines() { - let krate = json::from_str(line?.as_str())?; - out.push(krate); - } - out - }; - let found = krates - .iter_mut() - .find(|krate| krate.vers == version) - .ok_or_else(|| { - Error::from(AlexError::CrateNotFound { - name: String::from(name), - }) - })?; - - func(found); - - let lines = krates - .into_iter() - .map(|krate| json::to_string(&krate)) - .collect::, _>>()?; - fs::write(path.as_path(), lines.join("\n") + "\n")?; - - Ok(()) - } -} - #[derive(Debug, Clone, PartialEq)] struct Repository { path: PathBuf, diff --git a/src/index/mod.rs b/src/index/mod.rs index 84a98eeb..82c27f6f 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -3,6 +3,7 @@ use semver::{Version, VersionReq}; /// Index management through `git` shell command invocations. pub mod cli; mod models; +mod tree; /// Index management using [**`libgit2`**][libgit2]. /// [libgit2]: https://libgit2.org diff --git a/src/index/tree.rs b/src/index/tree.rs new file mode 100644 index 00000000..54d281b7 --- /dev/null +++ b/src/index/tree.rs @@ -0,0 +1,141 @@ +use super::models::CrateVersion; +use crate::error::AlexError; +use crate::Error; +use semver::{Version, VersionReq}; +use std::fs; +use std::io; +use std::io::BufRead; +use std::io::Write; +use std::path::PathBuf; + +#[derive(Debug, Clone, PartialEq)] +pub struct Tree { + path: PathBuf, +} + +impl Tree { + pub fn new(path: PathBuf) -> Self { + Self { path } + } + + fn compute_record_path(&self, name: &str) -> PathBuf { + match name.len() { + 1 => self.path.join("1").join(&name), + 2 => self.path.join("2").join(&name), + 3 => self.path.join("3").join(&name[..1]).join(&name), + _ => self.path.join(&name[0..2]).join(&name[2..4]).join(&name), + } + } + + pub fn match_record(&self, name: &str, req: VersionReq) -> Result { + let path = self.compute_record_path(name); + let file = fs::File::open(path).map_err(|err| match err.kind() { + io::ErrorKind::NotFound => Error::from(AlexError::CrateNotFound { + name: String::from(name), + }), + _ => Error::from(err), + })?; + let found = io::BufReader::new(file) + .lines() + .map(|line| Some(json::from_str(line.ok()?.as_str()).ok()?)) + .flat_map(|ret: Option| ret.into_iter()) + .filter(|krate| req.matches(&krate.vers)) + .max_by(|k1, k2| k1.vers.cmp(&k2.vers)); + Ok(found.ok_or_else(|| AlexError::CrateNotFound { + name: String::from(name), + })?) + } + + pub fn all_records(&self, name: &str) -> Result, Error> { + let path = self.compute_record_path(name); + let reader = io::BufReader::new(fs::File::open(path)?); + reader + .lines() + .map(|line| Ok(json::from_str::(line?.as_str())?)) + .collect() + } + + pub fn latest_record(&self, name: &str) -> Result { + let records = self.all_records(name)?; + Ok(records + .into_iter() + .max_by(|k1, k2| k1.vers.cmp(&k2.vers)) + .expect("at least one version should exist")) + } + + pub fn add_record(&self, record: CrateVersion) -> Result<(), Error> { + let path = self.compute_record_path(record.name.as_str()); + + if let Ok(file) = fs::File::open(&path) { + let reader = io::BufReader::new(file); + let records = reader + .lines() + .map(|line| Ok(json::from_str::(line?.as_str())?)) + .collect::, Error>>()?; + let latest = records + .into_iter() + .max_by(|k1, k2| k1.vers.cmp(&k2.vers)) + .expect("at least one record should exist"); + if record.vers <= latest.vers { + return Err(Error::from(AlexError::VersionTooLow { + krate: record.name, + hosted: latest.vers, + published: record.vers, + })); + } + } else { + let parent = path.parent().unwrap(); + fs::create_dir_all(parent)?; + } + + let mut file = fs::OpenOptions::new() + .write(true) + .append(true) + .create(true) + .open(path)?; + json::to_writer(&mut file, &record)?; + writeln!(file)?; + file.flush()?; + + Ok(()) + } + + pub fn alter_record(&self, name: &str, version: Version, func: F) -> Result<(), Error> + where + F: FnOnce(&mut CrateVersion), + { + let path = self.compute_record_path(name); + let file = fs::File::open(path.as_path()).map_err(|err| match err.kind() { + io::ErrorKind::NotFound => Error::from(AlexError::CrateNotFound { + name: String::from(name), + }), + _ => Error::from(err), + })?; + let mut krates: Vec = { + let mut out = Vec::new(); + for line in io::BufReader::new(file).lines() { + let krate = json::from_str(line?.as_str())?; + out.push(krate); + } + out + }; + let found = krates + .iter_mut() + .find(|krate| krate.vers == version) + .ok_or_else(|| { + Error::from(AlexError::CrateNotFound { + name: String::from(name), + }) + })?; + + func(found); + + let lines = krates + .into_iter() + .map(|krate| json::to_string(&krate)) + .collect::, _>>()?; + fs::write(path.as_path(), lines.join("\n") + "\n")?; + + Ok(()) + } +} From 681b10d0abcaf9ddee41a5ef66d8f8c48c546c7d Mon Sep 17 00:00:00 2001 From: Daniel Eades Date: Mon, 23 Mar 2020 20:13:54 +0000 Subject: [PATCH 5/5] refactor index::git2 to use 'Tree' --- src/index/git2.rs | 126 ++++++---------------------------------------- 1 file changed, 14 insertions(+), 112 deletions(-) diff --git a/src/index/git2.rs b/src/index/git2.rs index 71438fa4..918cfb2d 100644 --- a/src/index/git2.rs +++ b/src/index/git2.rs @@ -1,11 +1,10 @@ -use std::fs; -use std::io::{self, BufRead, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Mutex; use semver::{Version, VersionReq}; -use crate::error::{AlexError, Error}; +use crate::error::Error; +use crate::index::tree::Tree; use crate::index::{CrateVersion, Indexer}; /// The 'git2' crate index management strategy type. @@ -16,25 +15,17 @@ use crate::index::{CrateVersion, Indexer}; pub struct Git2Index { /// The path of the crate index. pub(crate) repo: Mutex, + tree: Tree, } impl Git2Index { /// Create a Git2Index instance with the given path. - pub fn new>(path: P) -> Result { - let repo = git2::Repository::open(path)?; + pub fn new>(path: P) -> Result { + let path = path.into(); + let repo = git2::Repository::open(&path)?; let repo = Mutex::new(repo); - Ok(Git2Index { repo }) - } - - fn compute_record_path(&self, name: &str) -> PathBuf { - let repo = self.repo.lock().unwrap(); - let path = repo.path().parent().unwrap(); - match name.len() { - 1 => path.join("1").join(&name), - 2 => path.join("2").join(&name), - 3 => path.join("3").join(&name[..1]).join(&name), - _ => path.join(&name[0..2]).join(&name[2..4]).join(&name), - } + let tree = Tree::new(path); + Ok(Git2Index { repo, tree }) } } @@ -145,114 +136,25 @@ impl Indexer for Git2Index { } fn match_record(&self, name: &str, req: VersionReq) -> Result { - let path = self.compute_record_path(name); - let file = fs::File::open(path).map_err(|err| match err.kind() { - io::ErrorKind::NotFound => Error::from(AlexError::CrateNotFound { - name: String::from(name), - }), - _ => Error::from(err), - })?; - let found = io::BufReader::new(file) - .lines() - .map(|line| Some(json::from_str(line.ok()?.as_str()).ok()?)) - .flat_map(|ret: Option| ret.into_iter()) - .filter(|krate| req.matches(&krate.vers)) - .max_by(|k1, k2| k1.vers.cmp(&k2.vers)); - Ok(found.ok_or_else(|| AlexError::CrateNotFound { - name: String::from(name), - })?) + self.tree.match_record(name, req) } fn all_records(&self, name: &str) -> Result, Error> { - let path = self.compute_record_path(name); - let reader = io::BufReader::new(fs::File::open(path)?); - reader - .lines() - .map(|line| Ok(json::from_str::(line?.as_str())?)) - .collect() + self.tree.all_records(name) } fn latest_record(&self, name: &str) -> Result { - let records = self.all_records(name)?; - Ok(records - .into_iter() - .max_by(|k1, k2| k1.vers.cmp(&k2.vers)) - .expect("at least one version should exist")) + self.tree.latest_record(name) } fn add_record(&self, record: CrateVersion) -> Result<(), Error> { - let path = self.compute_record_path(record.name.as_str()); - - if let Ok(file) = fs::File::open(&path) { - let reader = io::BufReader::new(file); - let records = reader - .lines() - .map(|line| Ok(json::from_str::(line?.as_str())?)) - .collect::, Error>>()?; - let latest = records - .into_iter() - .max_by(|k1, k2| k1.vers.cmp(&k2.vers)) - .expect("at least one record should exist"); - if record.vers <= latest.vers { - return Err(Error::from(AlexError::VersionTooLow { - krate: record.name, - hosted: latest.vers, - published: record.vers, - })); - } - } else { - let parent = path.parent().unwrap(); - fs::create_dir_all(parent)?; - } - - let mut file = fs::OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(path)?; - json::to_writer(&mut file, &record)?; - writeln!(file)?; - file.flush()?; - - Ok(()) + self.tree.add_record(record) } fn alter_record(&self, name: &str, version: Version, func: F) -> Result<(), Error> where F: FnOnce(&mut CrateVersion), { - let path = self.compute_record_path(name); - let file = fs::File::open(path.as_path()).map_err(|err| match err.kind() { - io::ErrorKind::NotFound => Error::from(AlexError::CrateNotFound { - name: String::from(name), - }), - _ => Error::from(err), - })?; - let mut krates: Vec = { - let mut out = Vec::new(); - for line in io::BufReader::new(file).lines() { - let krate = json::from_str(line?.as_str())?; - out.push(krate); - } - out - }; - let found = krates - .iter_mut() - .find(|krate| krate.vers == version) - .ok_or_else(|| { - Error::from(AlexError::CrateNotFound { - name: String::from(name), - }) - })?; - - func(found); - - let lines = krates - .into_iter() - .map(|krate| json::to_string(&krate)) - .collect::, _>>()?; - fs::write(path.as_path(), lines.join("\n") + "\n")?; - - Ok(()) + self.tree.alter_record(name, version, func) } }