Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement a --verify-hashes hash-checking mode #4007

Merged
merged 1 commit into from
Jul 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions crates/uv-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -803,13 +803,33 @@ pub struct PipSyncArgs {
/// - Editable installs are not supported.
/// - Local dependencies are not supported, unless they point to a specific wheel (`.whl`) or
/// source archive (`.zip`, `.tar.gz`), as opposed to a directory.
#[arg(long, env = "UV_REQUIRE_HASHES",
value_parser = clap::builder::BoolishValueParser::new(), overrides_with("no_require_hashes"))]
#[arg(
long,
env = "UV_REQUIRE_HASHES",
value_parser = clap::builder::BoolishValueParser::new(),
overrides_with("no_require_hashes"),
)]
pub require_hashes: bool,

#[arg(long, overrides_with("require_hashes"), hide = true)]
pub no_require_hashes: bool,

/// Validate any hashes provided in the requirements file.
///
/// Unlike `--require-hashes`, `--verify-hashes` does not require that all requirements have
/// hashes; instead, it will limit itself to verifying the hashes of those requirements that do
/// include them.
#[arg(
long,
env = "UV_VERIFY_HASHES",
value_parser = clap::builder::BoolishValueParser::new(),
overrides_with("no_verify_hashes"),
)]
pub verify_hashes: bool,

#[arg(long, overrides_with("verify_hashes"), hide = true)]
pub no_verify_hashes: bool,

/// The Python interpreter into which packages should be installed.
///
/// By default, uv installs into the virtual environment in the current working directory or
Expand Down Expand Up @@ -1084,6 +1104,22 @@ pub struct PipInstallArgs {
#[arg(long, overrides_with("require_hashes"), hide = true)]
pub no_require_hashes: bool,

/// Validate any hashes provided in the requirements file.
///
/// Unlike `--require-hashes`, `--verify-hashes` does not require that all requirements have
/// hashes; instead, it will limit itself to verifying the hashes of those requirements that do
/// include them.
#[arg(
long,
env = "UV_VERIFY_HASHES",
value_parser = clap::builder::BoolishValueParser::new(),
overrides_with("no_verify_hashes"),
)]
pub verify_hashes: bool,

#[arg(long, overrides_with("verify_hashes"), hide = true)]
pub no_verify_hashes: bool,

/// The Python interpreter into which packages should be installed.
///
/// By default, uv installs into the virtual environment in the current working directory or
Expand Down
35 changes: 35 additions & 0 deletions crates/uv-configuration/src/hash.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#[derive(Debug, Copy, Clone)]
pub enum HashCheckingMode {
/// Hashes should be validated against a pre-defined list of hashes. Every requirement must
/// itself be hashable (e.g., Git dependencies are forbidden) _and_ have a hash in the lockfile.
Require,
/// Hashes should be validated, if present, but ignored if absent.
Verify,
}

impl HashCheckingMode {
/// Return the [`HashCheckingMode`] from the command-line arguments, if any.
pub fn from_args(require_hashes: bool, verify_hashes: bool) -> Option<Self> {
if require_hashes {
Some(Self::Require)
} else if verify_hashes {
Some(Self::Verify)
} else {
None
}
}

/// Returns `true` if the hash checking mode is `Require`.
pub fn is_require(&self) -> bool {
matches!(self, Self::Require)
}
}

impl std::fmt::Display for HashCheckingMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Require => write!(f, "--require-hashes"),
Self::Verify => write!(f, "--verify-hashes"),
}
}
}
2 changes: 2 additions & 0 deletions crates/uv-configuration/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pub use concurrency::*;
pub use config_settings::*;
pub use constraints::*;
pub use extras::*;
pub use hash::*;
pub use name_specifiers::*;
pub use overrides::*;
pub use package_options::*;
Expand All @@ -16,6 +17,7 @@ mod concurrency;
mod config_settings;
mod constraints;
mod extras;
mod hash;
mod name_specifiers;
mod overrides;
mod package_options;
Expand Down
3 changes: 2 additions & 1 deletion crates/uv-requirements/src/source_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
let hashes = match self.hasher {
HashStrategy::None => HashPolicy::None,
HashStrategy::Generate => HashPolicy::Generate,
HashStrategy::Validate { .. } => {
HashStrategy::Verify(_) => HashPolicy::Generate,
HashStrategy::Require(_) => {
return Err(anyhow::anyhow!(
"Hash-checking is not supported for local directories: {}",
path.user_display()
Expand Down
13 changes: 13 additions & 0 deletions crates/uv-settings/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1018,6 +1018,19 @@ pub struct PipOptions {
"#
)]
pub require_hashes: Option<bool>,
/// Validate any hashes provided in the requirements file.
///
/// Unlike `--require-hashes`, `--verify-hashes` does not require that all requirements have
/// hashes; instead, it will limit itself to verifying the hashes of those requirements that do
/// include them.
#[option(
default = "false",
value_type = "bool",
example = r#"
verify-hashes = true
"#
)]
pub verify_hashes: Option<bool>,
/// Allow package upgrades, ignoring pinned versions in any existing output file.
#[option(
default = "false",
Expand Down
116 changes: 78 additions & 38 deletions crates/uv-types/src/hash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use url::Url;
use distribution_types::{DistributionMetadata, HashPolicy, PackageId, UnresolvedRequirement};
use pep508_rs::MarkerEnvironment;
use pypi_types::{HashDigest, HashError, Requirement, RequirementSource};
use uv_configuration::HashCheckingMode;
use uv_normalize::PackageName;

#[derive(Debug, Default, Clone)]
Expand All @@ -15,9 +16,14 @@ pub enum HashStrategy {
None,
/// Hashes should be generated (specifically, a SHA-256 hash), but not validated.
Generate,
/// Hashes should be validated against a pre-defined list of hashes. If necessary, hashes should
/// be generated so as to ensure that the archive is valid.
Validate(FxHashMap<PackageId, Vec<HashDigest>>),
/// Hashes should be validated, if present, but ignored if absent.
///
/// If necessary, hashes should be generated to ensure that the archive is valid.
Verify(FxHashMap<PackageId, Vec<HashDigest>>),
/// Hashes should be validated against a pre-defined list of hashes.
///
/// If necessary, hashes should be generated to ensure that the archive is valid.
Require(FxHashMap<PackageId, Vec<HashDigest>>),
}

impl HashStrategy {
Expand All @@ -26,7 +32,14 @@ impl HashStrategy {
match self {
Self::None => HashPolicy::None,
Self::Generate => HashPolicy::Generate,
Self::Validate(hashes) => HashPolicy::Validate(
Self::Verify(hashes) => {
if let Some(hashes) = hashes.get(&distribution.package_id()) {
HashPolicy::Validate(hashes.as_slice())
} else {
HashPolicy::None
}
}
Self::Require(hashes) => HashPolicy::Validate(
hashes
.get(&distribution.package_id())
.map(Vec::as_slice)
Expand All @@ -40,7 +53,14 @@ impl HashStrategy {
match self {
Self::None => HashPolicy::None,
Self::Generate => HashPolicy::Generate,
Self::Validate(hashes) => HashPolicy::Validate(
Self::Verify(hashes) => {
if let Some(hashes) = hashes.get(&PackageId::from_registry(name.clone())) {
HashPolicy::Validate(hashes.as_slice())
} else {
HashPolicy::None
}
}
Self::Require(hashes) => HashPolicy::Validate(
hashes
.get(&PackageId::from_registry(name.clone()))
.map(Vec::as_slice)
Expand All @@ -54,7 +74,14 @@ impl HashStrategy {
match self {
Self::None => HashPolicy::None,
Self::Generate => HashPolicy::Generate,
Self::Validate(hashes) => HashPolicy::Validate(
Self::Verify(hashes) => {
if let Some(hashes) = hashes.get(&PackageId::from_url(url)) {
HashPolicy::Validate(hashes.as_slice())
} else {
HashPolicy::None
}
}
Self::Require(hashes) => HashPolicy::Validate(
hashes
.get(&PackageId::from_url(url))
.map(Vec::as_slice)
Expand All @@ -68,7 +95,8 @@ impl HashStrategy {
match self {
Self::None => true,
Self::Generate => true,
Self::Validate(hashes) => hashes.contains_key(&PackageId::from_registry(name.clone())),
Self::Verify(_) => true,
Self::Require(hashes) => hashes.contains_key(&PackageId::from_registry(name.clone())),
}
}

Expand All @@ -77,7 +105,8 @@ impl HashStrategy {
match self {
Self::None => true,
Self::Generate => true,
Self::Validate(hashes) => hashes.contains_key(&PackageId::from_url(url)),
Self::Verify(_) => true,
Self::Require(hashes) => hashes.contains_key(&PackageId::from_url(url)),
}
}

Expand All @@ -90,6 +119,7 @@ impl HashStrategy {
pub fn from_requirements<'a>(
requirements: impl Iterator<Item = (&'a UnresolvedRequirement, &'a [String])>,
markers: Option<&MarkerEnvironment>,
mode: HashCheckingMode,
) -> Result<Self, HashStrategyError> {
let mut hashes = FxHashMap::<PackageId, Vec<HashDigest>>::default();

Expand All @@ -103,17 +133,25 @@ impl HashStrategy {
// Every requirement must be either a pinned version or a direct URL.
let id = match &requirement {
UnresolvedRequirement::Named(requirement) => {
uv_requirement_to_package_id(requirement)?
Self::pin(requirement).ok_or_else(|| {
HashStrategyError::UnpinnedRequirement(requirement.to_string(), mode)
})?
}
UnresolvedRequirement::Unnamed(requirement) => {
// Direct URLs are always allowed.
PackageId::from_url(&requirement.url.verbatim)
}
};

// Every requirement must include a hash.
if digests.is_empty() {
return Err(HashStrategyError::MissingHashes(requirement.to_string()));
// Under `--require-hashes`, every requirement must include a hash.
if mode.is_require() {
return Err(HashStrategyError::MissingHashes(
requirement.to_string(),
mode,
));
}
continue;
}

// Parse the hashes.
Expand All @@ -125,42 +163,44 @@ impl HashStrategy {
hashes.insert(id, digests);
}

Ok(Self::Validate(hashes))
match mode {
HashCheckingMode::Verify => Ok(Self::Verify(hashes)),
HashCheckingMode::Require => Ok(Self::Require(hashes)),
}
}
}

fn uv_requirement_to_package_id(requirement: &Requirement) -> Result<PackageId, HashStrategyError> {
Ok(match &requirement.source {
RequirementSource::Registry { specifier, .. } => {
// Must be a single specifier.
let [specifier] = specifier.as_ref() else {
return Err(HashStrategyError::UnpinnedRequirement(
requirement.to_string(),
));
};
/// Pin a [`Requirement`] to a [`PackageId`], if possible.
fn pin(requirement: &Requirement) -> Option<PackageId> {
match &requirement.source {
RequirementSource::Registry { specifier, .. } => {
// Must be a single specifier.
let [specifier] = specifier.as_ref() else {
return None;
};

// Must be pinned to a specific version.
if *specifier.operator() != pep440_rs::Operator::Equal {
return None;
}

// Must be pinned to a specific version.
if *specifier.operator() != pep440_rs::Operator::Equal {
return Err(HashStrategyError::UnpinnedRequirement(
requirement.to_string(),
));
Some(PackageId::from_registry(requirement.name.clone()))
}

PackageId::from_registry(requirement.name.clone())
RequirementSource::Url { url, .. }
| RequirementSource::Git { url, .. }
| RequirementSource::Path { url, .. }
| RequirementSource::Directory { url, .. } => Some(PackageId::from_url(url)),
}
RequirementSource::Url { url, .. }
| RequirementSource::Git { url, .. }
| RequirementSource::Path { url, .. }
| RequirementSource::Directory { url, .. } => PackageId::from_url(url),
})
}
}

#[derive(thiserror::Error, Debug)]
pub enum HashStrategyError {
#[error(transparent)]
Hash(#[from] HashError),
#[error("In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: {0}")]
UnpinnedRequirement(String),
#[error("In `--require-hashes` mode, all requirement must have a hash, but none were provided for: {0}")]
MissingHashes(String),
#[error(
"In `{1}` mode, all requirement must have their versions pinned with `==`, but found: {0}"
)]
UnpinnedRequirement(String, HashCheckingMode),
#[error("In `{1}` mode, all requirement must have a hash, but none were provided for: {0}")]
MissingHashes(String, HashCheckingMode),
}
9 changes: 5 additions & 4 deletions crates/uv/src/commands/pip/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, IndexStrategy, PreviewMode,
Reinstall, SetupPyStrategy, Upgrade,
BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, HashCheckingMode,
IndexStrategy, PreviewMode, Reinstall, SetupPyStrategy, Upgrade,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
Expand Down Expand Up @@ -52,7 +52,7 @@ pub(crate) async fn pip_install(
reinstall: Reinstall,
link_mode: LinkMode,
compile: bool,
require_hashes: bool,
hash_checking: Option<HashCheckingMode>,
setup_py: SetupPyStrategy,
connectivity: Connectivity,
config_settings: &ConfigSettings,
Expand Down Expand Up @@ -226,13 +226,14 @@ pub(crate) async fn pip_install(
let (tags, markers) = resolution_environment(python_version, python_platform, interpreter)?;

// Collect the set of required hashes.
let hasher = if require_hashes {
let hasher = if let Some(hash_checking) = hash_checking {
HashStrategy::from_requirements(
requirements
.iter()
.chain(overrides.iter())
.map(|entry| (&entry.requirement, entry.hashes.as_slice())),
Some(&markers),
hash_checking,
)?
} else {
HashStrategy::None
Expand Down
Loading
Loading