From b62d97fc3a89947137685152dca7fd2e00d3da5f Mon Sep 17 00:00:00 2001 From: konstin Date: Wed, 6 Dec 2023 11:30:43 +0100 Subject: [PATCH] Editables in pip-sync --- .gitignore | 15 ++- Cargo.lock | 5 + crates/distribution-types/src/direct_url.rs | 6 +- crates/install-wheel-rs/src/lib.rs | 17 +-- crates/install-wheel-rs/src/wheel.rs | 6 +- crates/puffin-cli/Cargo.toml | 3 + crates/puffin-cli/src/commands/pip_compile.rs | 87 +++++++++++++-- crates/puffin-cli/src/commands/pip_sync.rs | 83 ++++++++++++-- .../puffin-cli/src/commands/pip_uninstall.rs | 2 + crates/puffin-cli/src/requirements.rs | 19 +++- crates/puffin-cli/tests/pip_compile.rs | 51 +++++++++ crates/puffin-cli/tests/pip_sync.rs | 85 +++++++++++++++ crates/puffin-client/src/lib.rs | 4 +- crates/puffin-client/src/registry_client.rs | 74 ++++++------- crates/puffin-client/src/remote_metadata.rs | 2 +- crates/puffin-dev/src/resolve_cli.rs | 7 +- crates/puffin-dispatch/src/lib.rs | 12 ++- crates/puffin-installer/Cargo.toml | 1 + crates/puffin-installer/src/plan.rs | 33 ++++-- crates/puffin-installer/src/site_packages.rs | 18 +++- crates/puffin-resolver/Cargo.toml | 3 +- crates/puffin-resolver/src/manifest.rs | 16 +++ crates/puffin-resolver/src/resolution.rs | 43 +++++--- crates/puffin-resolver/src/resolver.rs | 30 +++++- crates/puffin-resolver/tests/resolver.rs | 101 +++++------------- crates/pypi-types/src/direct_url.rs | 7 +- crates/requirements-txt/src/lib.rs | 45 +++++++- .../maturin_editable/pyproject.toml | 2 +- .../python/maturin_editable/__init__.py | 6 +- .../poetry_editable/pyproject.toml | 1 + scripts/editable-installs/requirements.in | 3 + scripts/editable-installs/requirements.txt | 14 +++ 32 files changed, 602 insertions(+), 199 deletions(-) create mode 100644 scripts/editable-installs/requirements.in create mode 100644 scripts/editable-installs/requirements.txt diff --git a/.gitignore b/.gitignore index da5e94a6c32..79d27d5bf09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -.venv - # Generated by Cargo # will have compiled files and executables debug/ @@ -14,5 +12,16 @@ target/ # Use e.g. `--cache-dir cache-docker` to keep a cache across container invocations cache-* -# python tmp files +# Python tmp files __pycache__ + +# Maturin builds, and other native editable builds +*.so +*.pyd +*.dll + +# Profiling +flamegraph.svg +perf.data +perf.data.old +profile.json diff --git a/Cargo.lock b/Cargo.lock index 8f922b04a91..d58c1e0349a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2301,11 +2301,13 @@ dependencies = [ "chrono", "clap", "colored", + "distribution-filename", "distribution-types", "fs-err", "futures", "gourgeist", "indicatif", + "indoc", "insta", "insta-cmd", "install-wheel-rs", @@ -2318,6 +2320,7 @@ dependencies = [ "platform-tags", "predicates", "pubgrub", + "puffin-build", "puffin-cache", "puffin-client", "puffin-dispatch", @@ -2532,6 +2535,7 @@ dependencies = [ "puffin-traits", "pypi-types", "rayon", + "requirements-txt", "rustc-hash", "tempfile", "thiserror", @@ -2614,6 +2618,7 @@ dependencies = [ "puffin-normalize", "puffin-traits", "pypi-types", + "requirements-txt", "reqwest", "rustc-hash", "serde_json", diff --git a/crates/distribution-types/src/direct_url.rs b/crates/distribution-types/src/direct_url.rs index a25a6160c69..75e29c31f99 100644 --- a/crates/distribution-types/src/direct_url.rs +++ b/crates/distribution-types/src/direct_url.rs @@ -130,7 +130,7 @@ impl TryFrom<&LocalFileUrl> for pypi_types::DirectUrl { fn try_from(value: &LocalFileUrl) -> Result { Ok(pypi_types::DirectUrl::LocalDirectory { - url: value.url.to_string(), + url: value.url.clone(), dir_info: pypi_types::DirInfo { editable: None }, }) } @@ -141,7 +141,7 @@ impl TryFrom<&DirectArchiveUrl> for pypi_types::DirectUrl { fn try_from(value: &DirectArchiveUrl) -> Result { Ok(pypi_types::DirectUrl::ArchiveUrl { - url: value.url.to_string(), + url: value.url.clone(), archive_info: pypi_types::ArchiveInfo { hash: None, hashes: None, @@ -156,7 +156,7 @@ impl TryFrom<&DirectGitUrl> for pypi_types::DirectUrl { fn try_from(value: &DirectGitUrl) -> Result { Ok(pypi_types::DirectUrl::VcsUrl { - url: value.url.repository().to_string(), + url: value.url.repository().clone(), vcs_info: pypi_types::VcsInfo { vcs: pypi_types::VcsKind::Git, commit_id: value.url.precise().as_ref().map(ToString::to_string), diff --git a/crates/install-wheel-rs/src/lib.rs b/crates/install-wheel-rs/src/lib.rs index 181fac067a6..8961eaaec1e 100644 --- a/crates/install-wheel-rs/src/lib.rs +++ b/crates/install-wheel-rs/src/lib.rs @@ -92,6 +92,8 @@ pub enum Error { /// the metadata name and the dist info name are lowercase, while the wheel name is uppercase. /// Either way, we just search the wheel for the name. /// +/// Returns the dist info dir prefix with the `.dist-info` extension. +/// /// Reference implementation: pub fn find_dist_info<'a, T: Copy>( filename: &WheelFilename, @@ -106,13 +108,13 @@ pub fn find_dist_info<'a, T: Copy>( && Version::from_str(version).ok()? == filename.version && file == "METADATA" { - Some((payload, dist_info_dir)) + Some((payload, dir_stem)) } else { None } }) .collect(); - let (payload, dist_info_dir) = match metadatas[..] { + let (payload, dist_info_prefix) = match metadatas[..] { [] => { return Err(Error::MissingDistInfo); } @@ -127,7 +129,7 @@ pub fn find_dist_info<'a, T: Copy>( )); } }; - Ok((payload, dist_info_dir)) + Ok((payload, dist_info_prefix)) } /// Given an archive, read the `dist-info` metadata into a buffer. @@ -135,10 +137,11 @@ pub fn read_dist_info( filename: &WheelFilename, archive: &mut ZipArchive, ) -> Result, Error> { - let dist_info_dir = find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1; + let dist_info_prefix = + find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1; let mut file = archive - .by_name(&format!("{dist_info_dir}/METADATA")) + .by_name(&format!("{dist_info_prefix}.dist-info/METADATA")) .map_err(|err| Error::Zip(filename.to_string(), err))?; #[allow(clippy::cast_possible_truncation)] @@ -170,8 +173,8 @@ mod test { "Mastodon.py-1.5.1.dist-info/RECORD", ]; let filename = WheelFilename::from_str("Mastodon.py-1.5.1-py2.py3-none-any.whl").unwrap(); - let (_, dist_info_dir) = + let (_, dist_info_prefix) = find_dist_info(&filename, files.into_iter().map(|file| (file, file))).unwrap(); - assert_eq!(dist_info_dir, "Mastodon.py-1.5.1.dist-info"); + assert_eq!(dist_info_prefix, "Mastodon.py-1.5.1"); } } diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index b7a66448602..8d4173f842f 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -82,10 +82,10 @@ pub(crate) fn read_scripts_from_section( /// Extras are supposed to be ignored, which happens if you pass None for extras fn parse_scripts( archive: &mut ZipArchive, - dist_info_prefix: &str, + dist_info_dir: &str, extras: Option<&[String]>, ) -> Result<(Vec