Editables in pip-sync

.gitignore

@@ -1,5 +1,3 @@
-.venv
-
 # Generated by Cargo
 # will have compiled files and executables
 debug/
@@ -14,5 +12,16 @@ target/
 # Use e.g. `--cache-dir cache-docker` to keep a cache across container invocations
 cache-*
 
-# python tmp files
+# Python tmp files
 __pycache__
+
+# Maturin builds, and other native editable builds
+*.so
+*.pyd
+*.dll
+
+# Profiling
+flamegraph.svg
+perf.data
+perf.data.old
+profile.json

crates/distribution-types/src/direct_url.rs

@@ -130,7 +130,7 @@ impl TryFrom<&LocalFileUrl> for pypi_types::DirectUrl {
 
     fn try_from(value: &LocalFileUrl) -> Result<Self, Self::Error> {
         Ok(pypi_types::DirectUrl::LocalDirectory {
-            url: value.url.to_string(),
+            url: value.url.clone(),
             dir_info: pypi_types::DirInfo { editable: None },
         })
     }
@@ -141,7 +141,7 @@ impl TryFrom<&DirectArchiveUrl> for pypi_types::DirectUrl {
 
     fn try_from(value: &DirectArchiveUrl) -> Result<Self, Self::Error> {
         Ok(pypi_types::DirectUrl::ArchiveUrl {
-            url: value.url.to_string(),
+            url: value.url.clone(),
             archive_info: pypi_types::ArchiveInfo {
                 hash: None,
                 hashes: None,
@@ -156,7 +156,7 @@ impl TryFrom<&DirectGitUrl> for pypi_types::DirectUrl {
 
     fn try_from(value: &DirectGitUrl) -> Result<Self, Self::Error> {
         Ok(pypi_types::DirectUrl::VcsUrl {
-            url: value.url.repository().to_string(),
+            url: value.url.repository().clone(),
             vcs_info: pypi_types::VcsInfo {
                 vcs: pypi_types::VcsKind::Git,
                 commit_id: value.url.precise().as_ref().map(ToString::to_string),

crates/install-wheel-rs/src/lib.rs

@@ -92,6 +92,8 @@ pub enum Error {
 /// the metadata name and the dist info name are lowercase, while the wheel name is uppercase.
 /// Either way, we just search the wheel for the name.
 ///
+/// Returns the dist info dir prefix with the `.dist-info` extension.
+///
 /// Reference implementation: <https://github.com/pypa/packaging/blob/2f83540272e79e3fe1f5d42abae8df0c14ddf4c2/src/packaging/utils.py#L146-L172>
 pub fn find_dist_info<'a, T: Copy>(
     filename: &WheelFilename,
@@ -106,13 +108,13 @@ pub fn find_dist_info<'a, T: Copy>(
                 && Version::from_str(version).ok()? == filename.version
                 && file == "METADATA"
             {
-                Some((payload, dist_info_dir))
+                Some((payload, dir_stem))
             } else {
                 None
             }
         })
         .collect();
-    let (payload, dist_info_dir) = match metadatas[..] {
+    let (payload, dist_info_prefix) = match metadatas[..] {
         [] => {
             return Err(Error::MissingDistInfo);
         }
@@ -127,18 +129,19 @@ pub fn find_dist_info<'a, T: Copy>(
             ));
         }
     };
-    Ok((payload, dist_info_dir))
+    Ok((payload, dist_info_prefix))
 }
 
 /// Given an archive, read the `dist-info` metadata into a buffer.
 pub fn read_dist_info(
     filename: &WheelFilename,
     archive: &mut ZipArchive<impl Read + Seek + Sized>,
 ) -> Result<Vec<u8>, Error> {
-    let dist_info_dir = find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1;
+    let dist_info_prefix =
+        find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1;
 
     let mut file = archive
-        .by_name(&format!("{dist_info_dir}/METADATA"))
+        .by_name(&format!("{dist_info_prefix}.dist-info/METADATA"))
         .map_err(|err| Error::Zip(filename.to_string(), err))?;
 
     #[allow(clippy::cast_possible_truncation)]
@@ -170,8 +173,8 @@ mod test {
             "Mastodon.py-1.5.1.dist-info/RECORD",
         ];
         let filename = WheelFilename::from_str("Mastodon.py-1.5.1-py2.py3-none-any.whl").unwrap();
-        let (_, dist_info_dir) =
+        let (_, dist_info_prefix) =
             find_dist_info(&filename, files.into_iter().map(|file| (file, file))).unwrap();
-        assert_eq!(dist_info_dir, "Mastodon.py-1.5.1.dist-info");
+        assert_eq!(dist_info_prefix, "Mastodon.py-1.5.1");
     }
 }

crates/install-wheel-rs/src/wheel.rs

@@ -82,10 +82,10 @@ pub(crate) fn read_scripts_from_section(
 /// Extras are supposed to be ignored, which happens if you pass None for extras
 fn parse_scripts<R: Read + Seek>(
     archive: &mut ZipArchive<R>,
-    dist_info_prefix: &str,
+    dist_info_dir: &str,
     extras: Option<&[String]>,
 ) -> Result<(Vec<Script>, Vec<Script>), Error> {
-    let entry_points_path = format!("{dist_info_prefix}.dist-info/entry_points.txt");
+    let entry_points_path = format!("{dist_info_dir}/entry_points.txt");
     let entry_points_mapping = match archive.by_name(&entry_points_path) {
         Ok(mut file) => {
             let mut ini_text = String::new();
@@ -1043,7 +1043,7 @@ fn dist_info_metadata(
     archive: &mut ZipArchive<impl Read + Seek + Sized>,
 ) -> Result<Vec<u8>, Error> {
     let mut content = Vec::new();
-    let dist_info_file = format!("{dist_info_prefix}.dist-info/DIST-INFO");
+    let dist_info_file = format!("{dist_info_prefix}.dist-info/METADATA");
     archive
         .by_name(&dist_info_file)
         .map_err(|err| Error::Zip(dist_info_file.clone(), err))?

crates/puffin-cli/Cargo.toml

@@ -27,6 +27,8 @@ puffin-cache = { path = "../puffin-cache", features = ["clap"] }
 puffin-client = { path = "../puffin-client" }
 puffin-dispatch = { path = "../puffin-dispatch" }
 distribution-types = { path = "../distribution-types" }
+distribution-filename = { path = "../distribution-filename" }
+puffin-build = { path = "../puffin-build" }
 puffin-distribution = { path = "../puffin-distribution" }
 puffin-installer = { path = "../puffin-installer" }
 puffin-interpreter = { path = "../puffin-interpreter" }
@@ -71,6 +73,7 @@ tikv-jemallocator = "0.5.0"
 [dev-dependencies]
 assert_cmd = { version = "2.0.12" }
 assert_fs = { version = "1.0.13" }
+indoc = { version = "2.0.4" }
 insta-cmd = { version = "0.4.0" }
 insta = { version = "1.34.0", features = ["filters"] }
 predicates = { version = "3.0.4" }

crates/puffin-cli/src/commands/pip_compile.rs

@@ -1,25 +1,29 @@
 use std::borrow::Cow;
+use std::env;
 use std::fmt::Write;
 use std::io::stdout;
 use std::path::Path;
 use std::str::FromStr;
-use std::{env, fs};
 
 use anstream::AutoStream;
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
 use chrono::{DateTime, Utc};
 use colored::Colorize;
 use itertools::Itertools;
+use tempfile::tempdir_in;
 use tracing::debug;
 
-use pep508_rs::Requirement;
+use distribution_filename::WheelFilename;
+use distribution_types::{PathSourceDist, SourceDist};
+use pep508_rs::{Requirement, VersionOrUrl};
 use platform_host::Platform;
 use platform_tags::Tags;
+use puffin_build::{BuildKind, SourceBuild, SourceBuildContext};
 use puffin_cache::Cache;
-use puffin_client::RegistryClientBuilder;
+use puffin_client::{read_metadata_async, RegistryClientBuilder};
 use puffin_dispatch::BuildDispatch;
 use puffin_interpreter::Virtualenv;
-use puffin_normalize::ExtraName;
+use puffin_normalize::{ExtraName, PackageName};
 use puffin_resolver::{Manifest, PreReleaseMode, ResolutionMode, ResolutionOptions, Resolver};
 use pypi_types::IndexUrls;
 
@@ -74,8 +78,9 @@ pub(crate) async fn pip_compile(
     // Read all requirements from the provided sources.
     let RequirementsSpecification {
         project,
-        requirements,
+        mut requirements,
         constraints,
+        editables,
         extras: used_extras,
     } = RequirementsSpecification::try_from_sources(requirements, constraints, &extras)?;
 
@@ -108,7 +113,6 @@ pub(crate) async fn pip_compile(
         .unwrap_or_default();
 
     // Create a manifest of the requirements.
-    let manifest = Manifest::new(requirements, constraints, preferences, project);
     let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer);
 
     // Detect the current Python interpreter.
@@ -142,12 +146,75 @@ pub(crate) async fn pip_compile(
         client.clone(),
         cache.clone(),
         interpreter,
-        fs::canonicalize(venv.python_executable())?,
+        fs_err::canonicalize(venv.python_executable())?,
         no_build,
         index_urls,
     )
     .with_options(options);
 
+    // Build the editables and add their requirements
+    let mut editable_deps = Vec::new();
+    if !editables.is_empty() {
+        let reporter = ResolverReporter::from(printer);
+        for editable in editables {
+            debug!("Building (editable) {editable}");
+            let package_name = PackageName::new(format!(
+                "editable-at-{}",
+                editable
+                    .unwrap_path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+            ))
+            .unwrap();
+            let source_dist = SourceDist::Path(PathSourceDist {
+                name: package_name,
+                url: editable.url(),
+                path: editable.unwrap_path().to_path_buf(),
+            });
+            let idx = puffin_resolver::ResolverReporter::on_build_start(&reporter, &source_dist);
+            let editable_wheel_dir = tempdir_in(venv.root())?;
+            let builder = SourceBuild::setup(
+                editable.unwrap_path(),
+                None,
+                venv.interpreter(),
+                &build_dispatch,
+                SourceBuildContext::default(),
+                &editable.to_string(),
+                BuildKind::Editable,
+            )
+            .await
+            .context(format!("Failed to build editable {editable}"))?;
+            let wheel_name = builder.build(editable_wheel_dir.path()).await?;
+            let filename = WheelFilename::from_str(&wheel_name)
+                .context(format!("Invalid wheel filename for editable {editable}"))?;
+            let reader = fs_err::tokio::File::open(editable_wheel_dir.path().join(wheel_name))
+                .await
+                .context(format!("No wheel built for editable {editable}"))?;
+            let metadata = read_metadata_async(&filename, editable.to_string(), reader)
+                .await
+                .context(format!("Invalid wheel for editable {editable}"))?;
+
+            requirements.push(Requirement {
+                name: filename.name.clone(),
+                extras: None,
+                version_or_url: Some(VersionOrUrl::Url(editable.url().clone())),
+                marker: None,
+            });
+            editable_deps.push((editable.clone(), filename, metadata));
+            puffin_resolver::ResolverReporter::on_build_complete(&reporter, &source_dist, idx);
+        }
+    }
+
+    let manifest = Manifest::new(
+        requirements,
+        constraints,
+        preferences,
+        project,
+        editable_deps,
+    );
+
     // Resolve the dependencies.
     let resolver = Resolver::new(manifest, options, &markers, &tags, &client, &build_dispatch)
         .with_reporter(ResolverReporter::from(printer));
@@ -177,7 +244,9 @@ pub(crate) async fn pip_compile(
     )?;
 
     let mut writer: Box<dyn std::io::Write> = if let Some(output_file) = output_file {
-        Box::new(AutoStream::auto(fs::File::create(output_file)?))
+        Box::new(AutoStream::<std::fs::File>::auto(
+            fs_err::File::create(output_file)?.into(),
+        ))
     } else {
         Box::new(AutoStream::auto(stdout()))
     };