Remove build info by default and add explicit fetching (#917)

CHANGELOG.md

@@ -13,6 +13,11 @@ _Unreleased_
 
 - `rye test --quiet` no longer implies `--no-capture`. #915
 
+- Rye now can be used to fetch Python installations even when not using Rye
+  and build infos are no longer included by default.  This means that rather
+  than having interpreters at `~/.rye/py/cpython@3.11.1/install/bin/python3`
+  it will now reside at `~/.rye/py/cpython@3.11.1/bin/python3`.  #917
+
 <!-- released start -->
 
 ## 0.30.0

docs/guide/commands/fetch.md

@@ -3,6 +3,10 @@
 Fetches a Python interpreter for the local machine.  This command is
 available under the aliases `rye fetch` and `rye toolchain fetch`.
 
+As of Rye 0.31.0 toolchains are always fetched without build info.  This
+means that in the folder where toolchains are stored only the interpreter
+is found.  For more information see [Fetching Toolchains](../toolchains/index.md#build-info).
+
 ## Example
 
 Fetch a specific version of Python:
@@ -25,6 +29,13 @@ Unpacking
 Downloaded cpython@3.8.17
 ```
 
+To fetch a version of Python into a specific location rather than rye's
+interpreter cache:
+
+```
+$ rye fetch cpython@3.9.1 --target-path=my-interpreter
+```
+
 ## Arguments
 
 * `[VERSION]`: The version of Python to fetch.
@@ -35,6 +46,12 @@ Downloaded cpython@3.8.17
 
 * `-f, --force`: Fetch the Python toolchain even if it is already installed.
 
+* `--target-path` `<TARGET_PATH>`: Fetches the Python toolchain into an explicit location rather
+
+* `--build-info`: Fetches with build info
+
+* `--no-build-info`: Fetches without build info
+
 * `-v, --verbose`: Enables verbose diagnostics
 
 * `-q, --quiet`: Turns off all output

docs/guide/config.md

@@ -95,6 +95,11 @@ autosync = true
 # `false` to disable this behavior.
 venv-mark-sync-ignore = true
 
+# When set to `true` Rye will fetch certain interpreters with build information.
+# This will increase the space requirements, will put the interpreter into an
+# extra folder called `./install/` and place build artifacts adjacent in `./build`.
+fetch-with-build-info = false
+
 # a array of tables with optional sources.  Same format as in pyproject.toml
 [[sources]]
 name = "default"

docs/guide/toolchains/index.md

@@ -95,6 +95,19 @@ Toolchains are fetched from two sources:
 * [Indygreg's Portable Python Builds](https://github.com/indygreg/python-build-standalone) for CPython
 * [PyPy.org](https://www.pypy.org/) for PyPy
 
+You can also fetch toolchains into a specific location.  In this case the interpreter is not
+stored where Rye normally consults it, but in a specific location.  Rye will then not be able
+to use it unless it's manually registered.  This however can be useful for debugging or advanced
+setups:
+
+```
+rye toolchain fetch cpython@3.8.5 --target-path=my-interpreter
+```
+
+If you want to use rye interpreter fetching without installing rye, you might want to export the
+`RYE_NO_AUTO_INSTALL` environment variable and set it to `1` as otherwise the installer will kick
+in.
+
 ## Registering Toolchains
 
 Additionally, it's possible to register an external toolchain with the `rye toolchain register`
@@ -126,3 +139,19 @@ rye toolchain remove cpython@3.8.5
 !!! Warning
 
     Removing an actively used toolchain will render the virtualenvs that refer to use broken.
+
+## Build Info
+
++++ 0.31.0
+
+Prior to Rye 0.31.0 the Python installations were fetched with build infos.  You can see
+this because the folder structure in `~/.rye/py/INTERPRETER` is a bit different.  Rather than
+finding `cpython@3.8.5/bin/python3` there you will instead have an extra `install` folder
+(`cpython@3.8.5/install/bin/python3`) alongside a `build` folder containing the intermediate
+build outputs.  Starting with 0.31.0 the build info is removed by default.  If
+you want to get it back, you can explicitly fetch with `--build-info` or you can
+set the `behavior.fetch-with-build-info` config flag to true:
+
+```
+rye config --set-bool behavior.fetch-with-build-info=true
+```

rye/src/bootstrap.rs

@@ -9,6 +9,7 @@ use anyhow::{anyhow, bail, Context, Error};
 use console::style;
 use indicatif::{ProgressBar, ProgressStyle};
 use once_cell::sync::Lazy;
+use tempfile::tempdir_in;
 
 use crate::config::Config;
 use crate::piptools::LATEST_PIP;
@@ -303,7 +304,10 @@ fn ensure_latest_self_toolchain(output: CommandOutput) -> Result<PythonVersion,
         );
         Ok(version)
     } else {
-        fetch(&SELF_PYTHON_TARGET_VERSION, output, false)
+        fetch(
+            &SELF_PYTHON_TARGET_VERSION,
+            FetchOptions::with_output(output),
+        )
     }
 }
 
@@ -326,7 +330,7 @@ fn ensure_specific_self_toolchain(
             "Fetching requested internal toolchain '{}'",
             toolchain_version
         );
-        fetch(&toolchain_version.into(), output, false)
+        fetch(&toolchain_version.into(), FetchOptions::with_output(output))
     } else {
         echo!(
             if output,
@@ -337,66 +341,149 @@ fn ensure_specific_self_toolchain(
     }
 }
 
+/// Fetches a python installer.
+pub struct FetchOptions {
+    /// How verbose should the sync be?
+    pub output: CommandOutput,
+    /// Forces re-downloads even if they are already there.
+    pub force: bool,
+    /// Causes a fetch into a non standard location.
+    pub target_path: Option<PathBuf>,
+    /// Include build info (overrides configured default).
+    pub build_info: Option<bool>,
+}
+
+impl FetchOptions {
+    /// Basic fetch options.
+    pub fn with_output(output: CommandOutput) -> FetchOptions {
+        FetchOptions {
+            output,
+            ..Default::default()
+        }
+    }
+}
+
+impl Default for FetchOptions {
+    fn default() -> Self {
+        Self {
+            output: CommandOutput::Normal,
+            force: false,
+            target_path: None,
+            build_info: None,
+        }
+    }
+}
+
 /// Fetches a version if missing.
 pub fn fetch(
     version: &PythonVersionRequest,
-    output: CommandOutput,
-    force: bool,
+    options: FetchOptions,
 ) -> Result<PythonVersion, Error> {
-    if let Ok(version) = PythonVersion::try_from(version.clone()) {
-        let py_bin = get_toolchain_python_bin(&version)?;
-        if !force && py_bin.is_file() {
-            echo!(if verbose output, "Python version already downloaded. Skipping.");
-            return Ok(version);
-        }
-    }
-
     let (version, url, sha256) = match get_download_url(version) {
         Some(result) => result,
         None => bail!("unknown version {}", version),
     };
 
-    let target_dir = get_canonical_py_path(&version)?;
-    let target_py_bin = get_toolchain_python_bin(&version)?;
-    echo!(if verbose output, "target dir: {}", target_dir.display());
-    if target_dir.is_dir() && target_py_bin.is_file() {
-        if !force {
-            echo!(if verbose output, "Python version already downloaded. Skipping.");
-            return Ok(version);
+    let target_dir = match options.target_path {
+        Some(ref target_dir) => {
+            echo!(if options.output, "downloading to: {}", target_dir.display());
+            if target_dir.is_dir() {
+                if options.force {
+                    fs::remove_dir_all(target_dir)
+                        .path_context(target_dir, "could not remove target director")?;
+                } else {
+                    bail!("target directory '{}' exists", target_dir.display());
+                }
+            }
+            Cow::Borrowed(target_dir.as_path())
         }
-        echo!(if output, "Removing the existing Python version");
-        fs::remove_dir_all(&target_dir)
-            .with_context(|| format!("failed to remove target folder {}", target_dir.display()))?;
-    }
-
-    fs::create_dir_all(&target_dir).path_context(&target_dir, "failed to create target folder")?;
+        None => {
+            let target_dir = get_canonical_py_path(&version)?;
+            let target_py_bin = get_toolchain_python_bin(&version)?;
+            if target_dir.is_dir() && target_py_bin.is_file() {
+                if !options.force {
+                    echo!(if verbose options.output, "Python version already downloaded. Skipping.");
+                    return Ok(version);
+                }
+                echo!(if options.output, "Removing the existing Python version");
+                fs::remove_dir_all(&target_dir).with_context(|| {
+                    format!("failed to remove target folder {}", target_dir.display())
+                })?;
+            }
+            echo!(if verbose options.output, "target dir: {}", target_dir.display());
+            Cow::Owned(target_dir)
+        }
+    };
 
-    echo!(if verbose output, "download url: {}", url);
-    echo!(if output, "{} {}", style("Downloading").cyan(), version);
-    let archive_buffer = download_url(url, output)?;
+    echo!(if verbose options.output, "download url: {}", url);
+    echo!(if options.output, "{} {}", style("Downloading").cyan(), version);
+    let archive_buffer = download_url(url, options.output)?;
 
     if let Some(sha256) = sha256 {
-        echo!(if output, "{} {}", style("Checking").cyan(), "checksum");
+        echo!(if options.output, "{} {}", style("Checking").cyan(), "checksum");
         check_checksum(&archive_buffer, sha256)
             .with_context(|| format!("Checksum check of {} failed", &url))?;
     } else {
-        echo!(if output, "Checksum check skipped (no hash available)");
+        echo!(if options.output, "Checksum check skipped (no hash available)");
     }
 
-    echo!(if output, "{}", style("Unpacking").cyan());
-    unpack_archive(&archive_buffer, &target_dir, 1).with_context(|| {
+    echo!(if options.output, "{}", style("Unpacking").cyan());
+
+    let parent = target_dir
+        .parent()
+        .ok_or_else(|| anyhow!("cannot unpack to root"))?;
+    if !parent.exists() {
+        fs::create_dir_all(parent).path_context(&target_dir, "failed to create target folder")?;
+    }
+
+    let with_build_info = options
+        .build_info
+        .unwrap_or_else(|| Config::current().fetch_with_build_info());
+    let temp_dir = tempdir_in(target_dir.parent().unwrap()).context("temporary unpack location")?;
+
+    unpack_archive(&archive_buffer, temp_dir.path(), 1).with_context(|| {
         format!(
             "unpacking of downloaded tarball {} to '{}' failed",
             &url,
-            target_dir.display()
+            temp_dir.path().display(),
         )
     })?;
 
-    echo!(if output, "{} {}", style("Downloaded").green(), version);
+    // if we want to retain build infos or the installation has no build infos, then move
+    // the folder into the permanent location
+    if with_build_info || !installation_has_build_info(temp_dir.path()) {
+        let temp_dir = temp_dir.into_path();
+        fs::rename(&temp_dir, &target_dir).map_err(|err| {
+            fs::remove_dir_all(&temp_dir).ok();
+            err
+        })
+
+    // otherwise move the contents of the `install` folder over.
+    } else {
+        fs::rename(temp_dir.path().join("install"), &target_dir)
+    }
+    .path_context(&target_dir, "unable to persist download")?;
+
+    echo!(if options.output, "{} {}", style("Downloaded").green(), version);
 
     Ok(version)
 }
 
+fn installation_has_build_info(p: &Path) -> bool {
+    let mut has_install = false;
+    let mut has_build = false;
+    if let Ok(dir) = p.read_dir() {
+        for entry in dir.flatten() {
+            match entry.file_name().to_str() {
+                Some("install") => has_install = true,
+                Some("build") => has_build = true,
+                _ => {}
+            }
+        }
+    }
+    has_install && has_build
+}
+
 pub fn download_url(url: &str, output: CommandOutput) -> Result<Vec<u8>, Error> {
     match download_url_ignore_404(url, output)? {
         Some(result) => Ok(result),

rye/src/cli/fetch.rs

@@ -1,7 +1,9 @@
+use std::path::PathBuf;
+
 use anyhow::{Context, Error};
 use clap::Parser;
 
-use crate::bootstrap::fetch;
+use crate::bootstrap::{fetch, FetchOptions};
 use crate::config::Config;
 use crate::platform::get_python_version_request_from_pyenv_pin;
 use crate::pyproject::PyProject;
@@ -18,6 +20,15 @@ pub struct Args {
     /// Fetch the Python toolchain even if it is already installed.
     #[arg(short, long)]
     force: bool,
+    /// Fetches the Python toolchain into an explicit location rather.
+    #[arg(long)]
+    target_path: Option<PathBuf>,
+    /// Fetches with build info.
+    #[arg(long)]
+    build_info: bool,
+    /// Fetches without build info.
+    #[arg(long, conflicts_with = "build_info")]
+    no_build_info: bool,
     /// Enables verbose diagnostics.
     #[arg(short, long)]
     verbose: bool,
@@ -43,6 +54,21 @@ pub fn execute(cmd: Args) -> Result<(), Error> {
         }
     };
 
-    fetch(&version, output, cmd.force).context("error while fetching Python installation")?;
+    fetch(
+        &version,
+        FetchOptions {
+            output,
+            force: cmd.force,
+            target_path: cmd.target_path,
+            build_info: if cmd.build_info {
+                Some(true)
+            } else if cmd.no_build_info {
+                Some(false)
+            } else {
+                None
+            },
+        },
+    )
+    .context("error while fetching Python installation")?;
     Ok(())
 }

rye/src/config.rs

@@ -268,6 +268,17 @@ impl Config {
             .and_then(|x| x.as_bool())
             .unwrap_or(false)
     }
+
+    /// Fetches python installations with build info if possible.
+    ///
+    /// This used to be the default behavior in Rye prior to 0.31.
+    pub fn fetch_with_build_info(&self) -> bool {
+        self.doc
+            .get("behavior")
+            .and_then(|x| x.get("fetch-with-build-info"))
+            .and_then(|x| x.as_bool())
+            .unwrap_or(false)
+    }
 }
 
 #[cfg(test)]

rye/src/installer.rs

@@ -11,7 +11,7 @@ use regex::Regex;
 use same_file::is_same_file;
 use url::Url;
 
-use crate::bootstrap::{ensure_self_venv, fetch};
+use crate::bootstrap::{ensure_self_venv, fetch, FetchOptions};
 use crate::config::Config;
 use crate::consts::VENV_BIN;
 use crate::platform::get_app_dir;
@@ -131,7 +131,7 @@ pub fn install(
     uninstall_helper(&target_venv_path, &shim_dir)?;
 
     // make sure we have a compatible python version
-    let py_ver = fetch(py_ver, output, false)?;
+    let py_ver = fetch(py_ver, FetchOptions::with_output(output))?;
 
     create_virtualenv(
         output,