maturin run, a new subcommand that launches python with the correct venv

Cargo.toml

@@ -84,6 +84,9 @@ ureq = { version = "2.6.1", features = ["gzip", "socks-proxy"], default-features
 native-tls-crate = { package = "native-tls", version = "0.2.8", optional = true }
 keyring = { version = "1.1.1", optional = true }
 
+[target.'cfg(unix)'.dependencies]
+nix = { version = "0.26.2", default-features = false, features = ["process"], optional = true }
+
 [dev-dependencies]
 indoc = "2.0.0"
 pretty_assertions = "1.3.0"
@@ -93,7 +96,7 @@ trycmd = "0.14.11"
 which = "4.3.0"
 
 [features]
-default = ["full", "rustls"]
+default = ["full", "rustls", "maturin-run"]
 
 full = ["cross-compile", "log", "scaffolding", "upload"]
 
@@ -120,6 +123,9 @@ faster-tests = []
 # Deprecated features, keep it now for compatibility
 human-panic = []
 
+# Enables the `maturin run` subcommando, which may not work on all platforms
+maturin-run = ["nix"]
+
 # Without this, compressing the .gz archive becomes notably slow for debug builds
 [profile.dev.package.miniz_oxide]
 opt-level = 3

Changelog.md

@@ -12,11 +12,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * **Breaking Change**: Remove deprecated `python-source` option in `Cargo.toml` in [#1335](https://github.com/PyO3/maturin/pull/1335)
 * **Breaking Change**: Turn `patchelf` version warning into a hard error in [#1335](https://github.com/PyO3/maturin/pull/1335)
 * **Breaking Change**: [`uniffi_bindgen` CLI](https://mozilla.github.io/uniffi-rs/tutorial/Prerequisites.html#the-uniffi-bindgen-cli-tool) is required for building `uniffi` bindings wheels in [#1352](https://github.com/PyO3/maturin/pull/1352)
+* `maturin develop` now just like `maturin run` also look for a virtualenv .venv in the current or any parent directory if no environment is active.
 * Add Cargo compile targets configuration for filtering multiple bin targets in [#1339](https://github.com/PyO3/maturin/pull/1339)
 * Respect `rustflags` settings in cargo configuration file in [#1405](https://github.com/PyO3/maturin/pull/1405)
 * Bump MSRV to 1.63.0 in [#1407](https://github.com/PyO3/maturin/pull/1407)
 * Deprecate `--univeral2` in favor of `universal2-apple-darwin` target in [#1457](https://github.com/PyO3/maturin/pull/1457)
 * Raise an error when `Cargo.toml` contains removed python package metadata in [#1471](https://github.com/PyO3/maturin/pull/1471)
+* New subcommand: `maturin run <args>`. Equivalent to `python <args>`, except if neither a virtualenv nor a conda environment are activated, it looks for a virtualenv `.venv` in the current or any parent directory. This is inspired by [PEP 704](https://peps.python.org/pep-0704/). Note that on unix-like platforms this uses `execv` while on windows this uses a subcommand, for other platforms you can deactivate the `maturin-run` feature when building maturin.
 
 ## [0.14.12] - 2023-01-31
 

deny.toml

@@ -187,6 +187,7 @@ skip = [
     { name = "memoffset", version = "0.6.5" },
     { name = "proc-macro-crate", version = "0.1.5" },
     { name = "sha2", version = "0.9.9" },
+    { name = "nix", version = "0.22.3" },
 ]
 # Similarly to `skip` allows you to skip certain crates during duplicate
 # detection. Unlike skip, it also includes the entire tree of transitive

src/main.rs

@@ -16,8 +16,13 @@ use maturin::{
 #[cfg(feature = "upload")]
 use maturin::{upload_ui, PublishOpt};
 use std::env;
+#[cfg(unix)]
+use std::ffi::CString;
 use std::io;
 use std::path::PathBuf;
+#[cfg(windows)]
+use std::process::Command;
+use tracing::debug;
 
 #[derive(Debug, Parser)]
 #[command(
@@ -150,6 +155,12 @@ enum Opt {
         #[arg(value_name = "FILE")]
         files: Vec<PathBuf>,
     },
+    /// Run python from the virtualenv in `.venv` in the current or any parent folder
+    ///
+    /// On linux/mac, `maturin run <args>` is equivalent to `.venv/bin/python <args>`
+    #[cfg(feature = "maturin-run")]
+    #[structopt(external_subcommand)]
+    Run(Vec<String>),
     /// Backend for the PEP 517 integration. Not for human consumption
     ///
     /// The commands are meant to be called from the python PEP 517
@@ -211,6 +222,52 @@ enum Pep517Command {
     },
 }
 
+fn detect_venv(target: &Target) -> Result<PathBuf> {
+    match (env::var_os("VIRTUAL_ENV"), env::var_os("CONDA_PREFIX")) {
+        (Some(dir), None) => return Ok(PathBuf::from(dir)),
+        (None, Some(dir)) => return Ok(PathBuf::from(dir)),
+        (Some(_), Some(_)) => {
+            bail!("Both VIRTUAL_ENV and CONDA_PREFIX are set. Please unset one of them")
+        }
+        (None, None) => {
+            // No env var, try finding .venv
+        }
+    };
+
+    let current_dir = env::current_dir().context("Failed to detect current directory ಠ_ಠ")?;
+    // .venv in the current or any parent directory
+    for dir in current_dir.ancestors() {
+        let dot_venv = dir.join(".venv");
+        if dot_venv.is_dir() {
+            if !dot_venv.join("pyvenv.cfg").is_file() {
+                bail!(
+                    "Expected {} to be a virtual environment, but pyvenv.cfg is missing",
+                    dot_venv.display()
+                );
+            }
+            let python = target.get_venv_python(&dot_venv);
+            if !python.is_file() {
+                bail!(
+                    "Your virtualenv at {} is broken. It contains a pyvenv.cfg but no python at {}",
+                    dot_venv.display(),
+                    python.display()
+                );
+            }
+            debug!("Found a virtualenv named .venv at {}", dot_venv.display());
+            return Ok(dot_venv);
+        }
+    }
+
+    bail!(
+        "Couldn't find a virtualenv or conda environment, but you need one to use this command. \
+        For maturin to find your virtualenv you need to either set VIRTUAL_ENV (through activate), \
+        set CONDA_PREFIX (through conda activate) or have a virtualenv called .venv in the current \
+        or any parent folder. \
+        See https://virtualenv.pypa.io/en/latest/index.html on how to use virtualenv or \
+        use `maturin build` and `pip install <path/to/wheel>` instead."
+    )
+}
+
 /// Dispatches into the native implementations of the PEP 517 functions
 ///
 /// The last line of stdout is used as return value from the python part of the implementation
@@ -284,6 +341,64 @@ fn pep517(subcommand: Pep517Command) -> Result<()> {
     Ok(())
 }
 
+/// `maturin run` implementation. Looks for a virtualenv .venv in cwd or any parent and execve
+/// python from there with args
+#[cfg(feature = "maturin-run")]
+fn python_run(mut args: Vec<String>) -> Result<()> {
+    // Not sure if it's even feasible to support other target triples here given the restrictions
+    // with argument parsing
+    let target = Target::from_target_triple(None)?;
+    let venv_dir = detect_venv(&target)?;
+    let python = target.get_venv_python(venv_dir);
+
+    // We get the args in the format ["run", "arg1", "arg2"] but python shouldn't see the "run"
+    assert_eq!(args[0], "run");
+    args.remove(0);
+
+    #[cfg(unix)]
+    {
+        debug!("launching (execv) {}", python.display());
+        // Sorry for all the to_string_lossy
+        // https://stackoverflow.com/a/38948854/3549270
+        let executable_c_str = CString::new(python.to_string_lossy().as_bytes())
+            .context("Failed to convert executable path")?;
+        // Python needs first entry to be the binary (as it is common on unix) so python will
+        // pick up the pyvenv.cfg
+        args.insert(0, python.to_string_lossy().to_string());
+        let args_c_string = args
+            .iter()
+            .map(|arg| {
+                CString::new(arg.as_bytes()).context("Failed to convert executable argument")
+            })
+            .collect::<Result<Vec<CString>>>()?;
+
+        // We replace the current process with the new process is it's like actually just running
+        // the real thing.
+        // Note the that this may launch a python script, a native binary or anything else
+        nix::unistd::execv(&executable_c_str, &args_c_string)
+            .context("Failed to launch process")?;
+        unreachable!()
+    }
+    #[cfg(windows)]
+    {
+        debug!("launching (new process) {}", python.display());
+        // TODO: What's the correct equivalent of execv on windows?
+        let status = Command::new(python)
+            .args(args.iter())
+            .status()
+            .context("Failed to launch process")?;
+        std::process::exit(
+            status
+                .code()
+                .context("Process didn't return an exit code")?,
+        )
+    }
+    #[cfg(not(any(unix, windows)))]
+    {
+        compile_error!("Unsupported Platform, please disable the maturin-run feature.")
+    }
+}
+
 fn run() -> Result<()> {
     #[cfg(feature = "log")]
     tracing_subscriber::fmt::init();
@@ -400,6 +515,8 @@ fn run() -> Result<()> {
                 .build_source_distribution()?
                 .context("Failed to build source distribution, pyproject.toml not found")?;
         }
+        #[cfg(feature = "maturin-run")]
+        Opt::Run(args) => python_run(args)?,
         Opt::Pep517(subcommand) => pep517(subcommand)?,
         #[cfg(feature = "scaffolding")]
         Opt::InitProject { path, options } => init_project(path, options)?,

tests/common/other.rs

@@ -8,7 +8,11 @@ use std::fs::File;
 use std::io::Read;
 use std::iter::FromIterator;
 use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::str;
+use std::{env, fs};
 use tar::Archive;
+use tempfile::TempDir;
 use time::OffsetDateTime;
 use zip::ZipArchive;
 
@@ -22,7 +26,6 @@ pub fn test_musl() -> Result<bool> {
     use fs_err::File;
     use goblin::elf::Elf;
     use std::io::ErrorKind;
-    use std::process::Command;
 
     let get_target_list = Command::new("rustup")
         .args(["target", "list", "--installed"])
@@ -290,3 +293,59 @@ pub fn abi3_python_interpreter_args() -> Result<()> {
 
     Ok(())
 }
+
+/// Check that `maturin run` picks up a `.venv` in a parent directory
+pub fn maturin_run() -> Result<()> {
+    let main_dir = env::current_dir()?
+        .join("test-crates")
+        .join("venvs")
+        .join("maturin-run");
+    fs::create_dir_all(&main_dir)?;
+    // We can't use the same function as the other tests here because it needs to be called `.venv`
+    let venv_dir = main_dir.join(".venv");
+    if !venv_dir.is_dir() {
+        let status = Command::new("virtualenv").arg(&venv_dir).status()?;
+        assert!(status.success());
+    }
+
+    let work_dir = main_dir.join("some_work_dir");
+    fs::create_dir_all(&work_dir)?;
+
+    let output = Command::new(env!("CARGO_BIN_EXE_maturin"))
+        .args(["run", "-c", "import sys; print(sys.prefix)"])
+        .current_dir(&work_dir)
+        .output()
+        .expect("Failed to launch maturin");
+    if !output.status.success() {
+        panic!(
+            "`maturin run` failed: {}\n---stdout:\n{}---stderr:\n{}",
+            output.status,
+            str::from_utf8(&output.stdout)?,
+            str::from_utf8(&output.stderr)?
+        );
+    }
+
+    // Check that the prefix ends with .venv, i.e. that we found the right venv
+    assert_eq!(
+        Path::new(&String::from_utf8(output.stdout)?.trim()),
+        &venv_dir
+    );
+
+    Ok(())
+}
+
+/// Check that `maturin run` fails when there is no `.venv`
+pub fn maturin_run_error() -> Result<()> {
+    let temp_dir = TempDir::new()?;
+
+    let output = Command::new(env!("CARGO_BIN_EXE_maturin"))
+        .args(["run", "-c", "import sys; print(sys.prefix)"])
+        .current_dir(temp_dir.path())
+        .output()
+        .expect("Failed to launch maturin");
+    assert!(!output.status.success());
+    // Don't check the exact error message but make sure it tells the user about .venv
+    assert!(str::from_utf8(&output.stderr)?.contains(".venv"));
+
+    Ok(())
+}

tests/run.rs

@@ -613,3 +613,13 @@ fn pyo3_source_date_epoch() {
         "pyo3_source_date_epoch",
     ))
 }
+
+#[test]
+fn maturin_run() {
+    handle_result(other::maturin_run())
+}
+
+#[test]
+fn maturin_run_error() {
+    handle_result(other::maturin_run_error())
+}