From 6f305a2ff6771013a381f37abc237372025bc561 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Mon, 13 May 2024 17:46:30 +0200 Subject: [PATCH 01/34] misc: add name to info --- src/cli/info.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cli/info.rs b/src/cli/info.rs index 93d15565e..4c6be623e 100644 --- a/src/cli/info.rs +++ b/src/cli/info.rs @@ -37,6 +37,7 @@ pub struct Args { #[derive(Serialize)] pub struct ProjectInfo { + name: String, manifest_path: PathBuf, last_updated: Option, pixi_folder_size: Option, @@ -210,6 +211,7 @@ impl Display for Info { if let Some(pi) = self.project_info.as_ref() { writeln!(f, "\n{}", bold.apply_to("Project\n------------"))?; + writeln!(f, "{:>WIDTH$}: {}", bold.apply_to("Name"), pi.name)?; if let Some(version) = pi.version.clone() { writeln!(f, "{:>WIDTH$}: {}", bold.apply_to("Version"), version)?; } @@ -288,7 +290,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { let project = Project::load_or_else_discover(args.manifest_path.as_deref()).ok(); let (pixi_folder_size, cache_size) = if args.extended { - let env_dir = project.as_ref().map(|p| p.root().join(".pixi")); + let env_dir = project.as_ref().map(|p| p.pixi_dir()); let cache_dir = config::get_cache_dir()?; await_in_progress("fetching directory sizes", |_| { spawn_blocking(move || { @@ -304,6 +306,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { }; let project_info = project.clone().map(|p| ProjectInfo { + name: p.name().to_string(), manifest_path: p.manifest_path(), last_updated: last_updated(p.lock_file_path()).ok(), pixi_folder_size, From 77f59847c726895a75bc8ba317a026b0cbb1f98f Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Mon, 13 May 2024 18:08:01 +0200 Subject: [PATCH 02/34] feat: add target-environments-dir --- src/config.rs | 61 ++++++++++++++++++++++++++++++++------ src/project/mod.rs | 15 ++++++++++ tests/config/config_1.toml | 1 + tests/config/config_2.toml | 2 ++ 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/src/config.rs b/src/config.rs index ee35e7c92..db0af1a2f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,7 +1,7 @@ use clap::{ArgAction, Parser}; -use miette::{Context, IntoDiagnostic}; +use miette::{miette, Context, IntoDiagnostic}; use rattler_conda_types::{Channel, ChannelConfig, ParseChannelError}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -108,7 +108,7 @@ pub struct ConfigCliPrompt { change_ps1: Option, } -#[derive(Clone, Default, Debug, Deserialize)] +#[derive(Clone, Default, Debug, Deserialize, Serialize)] pub struct RepodataConfig { /// Disable JLAP compression for repodata. #[serde(alias = "disable-jlap")] // BREAK: rename instead of alias @@ -121,14 +121,14 @@ pub struct RepodataConfig { pub disable_zstd: Option, } -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, clap::ValueEnum)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, clap::ValueEnum)] #[serde(rename_all = "lowercase")] pub enum KeyringProvider { Disabled, Subprocess, } -#[derive(Clone, Debug, Deserialize, Default)] +#[derive(Clone, Debug, Deserialize, Serialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] pub struct PyPIConfig { /// The default index URL for PyPI packages. @@ -172,7 +172,7 @@ impl PyPIConfig { } } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct Config { #[serde(default)] #[serde(alias = "default-channels")] // BREAK: rename instead of alias @@ -212,6 +212,14 @@ pub struct Config { #[serde(default)] #[serde(rename = "pypi-config")] pub pypi_config: PyPIConfig, + + /// The location of the environments build by pixi + #[serde(default)] + #[serde( + alias = "target-environments-directory", + skip_serializing_if = "Option::is_none" + )] + target_environments_directory: Option, } impl Default for Config { @@ -226,6 +234,7 @@ impl Default for Config { channel_config: default_channel_config(), repodata_config: None, pypi_config: PyPIConfig::default(), + target_environments_directory: None, } } } @@ -261,9 +270,22 @@ impl Config { config.loaded_from.push(location.to_path_buf()); + config.validate()?; + Ok(config) } + /// Validate the config file. + pub fn validate(&self) -> miette::Result<()> { + if let Some(target_env_dir) = self.target_environments_directory.clone() { + if !target_env_dir.exists() { + return Err(miette!("The `target-environments-directory` path does not exist: {:?}. It needs to be an absolute path to a directory.", target_env_dir)); + } + } + + Ok(()) + } + /// Load the global config file from the home directory (~/.pixi/config.toml) pub fn load_global() -> Config { #[cfg(target_os = "windows")] @@ -357,6 +379,9 @@ impl Config { channel_config: other.channel_config, repodata_config: other.repodata_config.or(self.repodata_config), pypi_config: other.pypi_config.merge(self.pypi_config), + target_environments_directory: other + .target_environments_directory + .or(self.target_environments_directory), } } @@ -414,6 +439,11 @@ impl Config { pub fn mirror_map(&self) -> &std::collections::HashMap> { &self.mirrors } + + /// Retrieve the value for the target_environments_directory field. + pub fn target_environments_directory(&self) -> Option { + self.target_environments_directory.clone() + } } #[cfg(test)] @@ -422,13 +452,21 @@ mod tests { #[test] fn test_config_parse() { - let toml = r#" + let toml = format!( + r#" default_channels = ["conda-forge"] tls_no_verify = true - "#; - let config = Config::from_toml(toml, &PathBuf::from("")).unwrap(); + target-environments-directory = "{}" + "#, + env!("CARGO_MANIFEST_DIR") + ); + let config = Config::from_toml(toml.as_str(), &PathBuf::from("")).unwrap(); assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(true)); + assert_eq!( + config.target_environments_directory, + Some(PathBuf::from(env!("CARGO_MANIFEST_DIR"))) + ); } #[test] @@ -486,11 +524,16 @@ mod tests { default_channels: vec!["conda-forge".to_string()], channel_config: ChannelConfig::default_with_root_dir(PathBuf::from("/root/dir")), tls_no_verify: Some(true), + target_environments_directory: Some(PathBuf::from("/path/to/envs")), ..Default::default() }; config = config.merge_config(other); assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(true)); + assert_eq!( + config.target_environments_directory, + Some(PathBuf::from("/path/to/envs")) + ); let d = Path::new(&env!("CARGO_MANIFEST_DIR")) .join("tests") diff --git a/src/project/mod.rs b/src/project/mod.rs index 136a8f3f3..76e6c8988 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -23,6 +23,7 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; +use xxhash_rust::xxh3::xxh3_64; use crate::activation::{get_environment_variables, run_activation}; use crate::config::Config; @@ -207,6 +208,7 @@ impl Project { let env_vars = Project::init_env_vars(&manifest.parsed.environments); + // Load the user configuration from the local project and all default locations let config = Config::load(&root.join(consts::PIXI_DIR))?; let (client, authenticated_client) = build_reqwest_clients(Some(&config)); @@ -283,6 +285,19 @@ impl Project { /// Returns the pixi directory pub fn pixi_dir(&self) -> PathBuf { + // Custom root directory for target environments if set in configuration. + if let Some(custom_root) = self.config().target_environments_directory() { + tracing::info!( + "Using custom target directory for environments: {}", + custom_root.display() + ); + return custom_root.join(format!( + "{}-{}", + self.name(), + xxh3_64(self.root.to_string_lossy().as_bytes()) + )); + } + tracing::debug!("Using default root directory for target environments"); self.root.join(consts::PIXI_DIR) } diff --git a/tests/config/config_1.toml b/tests/config/config_1.toml index 7ec6a40d0..9c626034b 100644 --- a/tests/config/config_1.toml +++ b/tests/config/config_1.toml @@ -1,2 +1,3 @@ default_channels = ["conda-forge", "bioconda", "defaults"] tls_no_verify = true +target-environments-directory = "/home/test/envs" diff --git a/tests/config/config_2.toml b/tests/config/config_2.toml index b4455099d..25229e13b 100644 --- a/tests/config/config_2.toml +++ b/tests/config/config_2.toml @@ -4,3 +4,5 @@ change_ps1 = true [repodata_config] disable_jlap = true disable_zstd = true + +target-environments-directory = "/etc/conda/pixi.d/environments.d" From 8e884112315918d3344c5cf402b82edd6b014130 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 10:07:34 +0200 Subject: [PATCH 03/34] fix: tests --- src/config.rs | 16 ++++++++++++++++ .../pixi__config__tests__config_merge.snap | 1 + tests/config/config_1.toml | 1 - tests/config/config_2.toml | 2 -- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/config.rs b/src/config.rs index db0af1a2f..36181589e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -535,6 +535,22 @@ mod tests { Some(PathBuf::from("/path/to/envs")) ); + let other2 = Config { + default_channels: vec!["channel".to_string()], + channel_config: ChannelConfig::default_with_root_dir(PathBuf::from("/root/dir2")), + tls_no_verify: Some(false), + target_environments_directory: Some(PathBuf::from("/path/to/envs2")), + ..Default::default() + }; + + config = config.merge_config(other2); + assert_eq!(config.default_channels, vec!["channel"]); + assert_eq!(config.tls_no_verify, Some(false)); + assert_eq!( + config.target_environments_directory, + Some(PathBuf::from("/path/to/envs2")) + ); + let d = Path::new(&env!("CARGO_MANIFEST_DIR")) .join("tests") .join("config"); diff --git a/src/snapshots/pixi__config__tests__config_merge.snap b/src/snapshots/pixi__config__tests__config_merge.snap index e9061025f..86043da8f 100644 --- a/src/snapshots/pixi__config__tests__config_merge.snap +++ b/src/snapshots/pixi__config__tests__config_merge.snap @@ -54,4 +54,5 @@ Config { extra_index_urls: [], keyring_provider: None, }, + target_environments_directory: None, } diff --git a/tests/config/config_1.toml b/tests/config/config_1.toml index 9c626034b..7ec6a40d0 100644 --- a/tests/config/config_1.toml +++ b/tests/config/config_1.toml @@ -1,3 +1,2 @@ default_channels = ["conda-forge", "bioconda", "defaults"] tls_no_verify = true -target-environments-directory = "/home/test/envs" diff --git a/tests/config/config_2.toml b/tests/config/config_2.toml index 25229e13b..b4455099d 100644 --- a/tests/config/config_2.toml +++ b/tests/config/config_2.toml @@ -4,5 +4,3 @@ change_ps1 = true [repodata_config] disable_jlap = true disable_zstd = true - -target-environments-directory = "/etc/conda/pixi.d/environments.d" From 14aef4271230b3e30fa484ca046c61084b9ec87b Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 11:20:27 +0200 Subject: [PATCH 04/34] test: extended the tests with an installation in a target folder --- src/config.rs | 2 +- tests/install_tests.rs | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 36181589e..4d004bd9a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -219,7 +219,7 @@ pub struct Config { alias = "target-environments-directory", skip_serializing_if = "Option::is_none" )] - target_environments_directory: Option, + pub target_environments_directory: Option, } impl Default for Config { diff --git a/tests/install_tests.rs b/tests/install_tests.rs index b4c064472..ceac39717 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -1,13 +1,18 @@ mod common; +use std::fs::{create_dir_all, File}; +use std::io::Write; use std::path::Path; use std::str::FromStr; use crate::common::builders::string_from_iter; use crate::common::package_database::{Package, PackageDatabase}; use common::{LockFileExt, PixiControl}; +use pixi::cli::run::Args; use pixi::cli::{run, LockFileUsageArgs}; +use pixi::config::Config; use pixi::consts::{DEFAULT_ENVIRONMENT_NAME, PIXI_UV_INSTALLER}; +use pixi::FeatureName; use rattler_conda_types::Platform; use serial_test::serial; use tempfile::TempDir; @@ -120,9 +125,24 @@ async fn test_incremental_lock_file() { #[tokio::test] #[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] -async fn install_locked() { +async fn install_locked_with_config() { let pixi = PixiControl::new().unwrap(); pixi.init().await.unwrap(); + + // Overwrite install location to a target directory + let mut config = Config::default(); + let target_dir = pixi.project_path().join("target"); + config.target_environments_directory = Some(target_dir.clone()); + create_dir_all(target_dir.clone()).unwrap(); + dbg!(config.clone().target_environments_directory); + + let config_path = pixi.project().unwrap().pixi_dir().join("config.toml"); + create_dir_all(config_path.parent().unwrap()).unwrap(); + + let mut file = File::create(config_path).unwrap(); + file.write_all(toml::to_string(&config).unwrap().as_bytes()) + .unwrap(); + // Add and update lockfile with this version of python let python_version = if cfg!(target_os = "macos") && cfg!(target_arch = "aarch64") { "python==3.10.0" @@ -163,6 +183,24 @@ async fn install_locked() { Platform::current(), "python==3.9.0" )); + + // Verify that the folders are present in the target directory using a task. + pixi.tasks() + .add("which_python".into(), None, FeatureName::Default) + .with_commands(["which python"]) + .execute() + .unwrap(); + + let result = pixi + .run(Args { + task: vec!["which_python".to_string()], + manifest_path: None, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(result.exit_code, 0); + assert!(result.stdout.contains(target_dir.to_str().unwrap())); } /// Test `pixi install/run --frozen` functionality From 6e74e9ae932b079de245a3fdf95ea654145f7e55 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 13:27:40 +0200 Subject: [PATCH 05/34] docs: document the new config option and refactor the whole config documentation --- docs/advanced/authentication.md | 4 +- docs/advanced/explain_info_command.md | 4 +- docs/advanced/global_configuration.md | 155 ------------- docs/advanced/pyproject_toml.md | 2 +- docs/features/advanced_tasks.md | 2 +- docs/features/lockfile.md | 4 +- docs/reference/cli.md | 70 +++--- docs/reference/pixi_configuration.md | 209 ++++++++++++++++++ ...figuration.md => project_configuration.md} | 2 +- docs/tutorials/python.md | 2 +- docs/tutorials/ros2.md | 2 +- mkdocs.yml | 7 +- src/config.rs | 2 +- 13 files changed, 260 insertions(+), 205 deletions(-) delete mode 100644 docs/advanced/global_configuration.md create mode 100644 docs/reference/pixi_configuration.md rename docs/reference/{configuration.md => project_configuration.md} (99%) diff --git a/docs/advanced/authentication.md b/docs/advanced/authentication.md index 071478f81..91d16fa64 100644 --- a/docs/advanced/authentication.md +++ b/docs/advanced/authentication.md @@ -102,7 +102,7 @@ The JSON should follow the following format: Note: if you use a wildcard in the host, any subdomain will match (e.g. `*.prefix.dev` also matches `repo.prefix.dev`). -Lastly you can set the authentication override file in the [global configuration file](./global_configuration.md). +Lastly you can set the authentication override file in the [global configuration file](./../reference/pixi_configuration.md). ## PyPI authentication Currently, we support the following methods for authenticating against PyPI: @@ -122,7 +122,7 @@ To enable this use the CLI flag `--pypi-keyring-provider` which can either be se pixi install --pypi-keyring-provider subprocess ``` -This option can also be set in the global configuration file under [pypi-config](./global_configuration.md#pypi-configuration). +This option can also be set in the global configuration file under [pypi-config](./../reference/pixi_configuration.md#pypi-configuration). #### Installing keyring To install keyring you can use pixi global install: diff --git a/docs/advanced/explain_info_command.md b/docs/advanced/explain_info_command.md index 2d2a3e260..ba874f199 100644 --- a/docs/advanced/explain_info_command.md +++ b/docs/advanced/explain_info_command.md @@ -83,11 +83,11 @@ The size of the previously mentioned "Cache dir" in Mebibytes. ## Project info Everything below `Project` is info about the project you're currently in. -This info is only available if your path has a [manifest file](../reference/configuration.md). +This info is only available if your path has a [manifest file](../reference/project_configuration.md). ### Manifest file -The path to the [manifest file](../reference/configuration.md) that describes the project. +The path to the [manifest file](../reference/project_configuration.md) that describes the project. ### Last updated diff --git a/docs/advanced/global_configuration.md b/docs/advanced/global_configuration.md deleted file mode 100644 index 2498d5892..000000000 --- a/docs/advanced/global_configuration.md +++ /dev/null @@ -1,155 +0,0 @@ -# Global configuration in pixi - -Pixi supports some global configuration options, as well as project-scoped -configuration (that does not belong into the project file). The configuration is -loaded in the following order: - -1. System configuration folder (`/etc/pixi/config.toml` or `C:\ProgramData\pixi\config.toml`) -2. XDG compliant configuration folder (`$XDG_CONFIG_HOME/pixi/config.toml` or - `$HOME/.config/pixi/config.toml`) -3. Global configuration folder, depending on the OS: - - Linux: `$HOME/.config/pixi/config.toml` - - macOS: `$HOME/Library/Application Support/pixi/config.toml` - - Windows: `%APPDATA%\pixi\config.toml` -4. Global .pixi folder: `~/.pixi/config.toml` (or `$PIXI_HOME/config.toml` if - the `PIXI_HOME` environment variable is set) -5. Project-local .pixi folder: `$PIXI_PROJECT/.pixi/config.toml` -6. Command line arguments (`--tls-no-verify`, `--change-ps1=false` etc.) - -!!! note - To find the locations where `pixi` looks for configuration files, run - `pixi` with `-v` or `--verbose`. - -## Reference - -??? info "Casing In Configuration" - In versions of pixi `0.20.1` and older the global configuration used snake_case - we've changed to `kebab-case` for consistency with the rest of the configuration. - But we still support the old `snake_case` configuration, for older configuration options. - These are: - - - `default_channels` - - `change_ps1` - - `tls_no_verify` - - `authentication_override_file` - - `mirrors` and sub-options - - `repodata-config` and sub-options - -The following reference describes all available configuration options. - -```toml -# The default channels to select when running `pixi init` or `pixi global install`. -# This defaults to only conda-forge. -default-channels = ["conda-forge"] - -# When set to false, the `(pixi)` prefix in the shell prompt is removed. -# This applies to the `pixi shell` subcommand. -# You can override this from the CLI with `--change-ps1`. -change-ps1 = true - -# When set to true, the TLS certificates are not verified. Note that this is a -# security risk and should only be used for testing purposes or internal networks. -# You can override this from the CLI with `--tls-no-verify`. -tls-no-verify = false - -# Override from where the authentication information is loaded. -# Usually we try to use the keyring to load authentication data from, and only use a JSON -# file as fallback. This option allows you to force the use of a JSON file. -# Read more in the authentication section. -authentication-override-file = "/path/to/your/override.json" - -# configuration for conda channel-mirrors -[mirrors] -# redirect all requests for conda-forge to the prefix.dev mirror -"https://conda.anaconda.org/conda-forge" = [ - "https://prefix.dev/conda-forge" -] - -# redirect all requests for bioconda to one of the three listed mirrors -# Note: for repodata we try the first mirror first. -"https://conda.anaconda.org/bioconda" = [ - "https://conda.anaconda.org/bioconda", - # OCI registries are also supported - "oci://ghcr.io/channel-mirrors/bioconda", - "https://prefix.dev/bioconda", -] - -[repodata-config] -# disable fetching of jlap, bz2 or zstd repodata files. -# This should only be used for specific old versions of artifactory and other non-compliant -# servers. -disable-jlap = true # don't try to download repodata.jlap -disable-bzip2 = true # don't try to download repodata.json.bz2 -disable-zstd = true # don't try to download repodata.json.zst - - -[pypi-config] -# These are sections specifically related to the PyPI configuration. -index-url = "https://pypi.org/simple" -extra-index-urls = ["https://pypi.org/simple2"] -keyring-provider = "subprocess" -``` - -## Mirror configuration - -You can configure mirrors for conda channels. We expect that mirrors are exact -copies of the original channel. The implementation will look for the mirror key -(a URL) in the `mirrors` section of the configuration file and replace the -original URL with the mirror URL. - -To also include the original URL, you have to repeat it in the list of mirrors. - -The mirrors are prioritized based on the order of the list. We attempt to fetch -the repodata (the most important file) from the first mirror in the list. The -repodata contains all the SHA256 hashes of the individual packages, so it is -important to get this file from a trusted source. - -You can also specify mirrors for an entire "host", e.g. - -```toml -[mirrors] -"https://conda.anaconda.org" = [ - "https://prefix.dev/" -] -``` - -This will forward all request to channels on anaconda.org to prefix.dev. -Channels that are not currently mirrored on prefix.dev will fail in the above example. - -### OCI Mirrors - -You can also specify mirrors on the OCI registry. There is a public mirror on -the Github container registry (ghcr.io) that is maintained by the conda-forge -team. You can use it like this: - -```toml -[mirrors] -"https://conda.anaconda.org/conda-forge" = [ - "oci://ghcr.io/channel-mirrors/conda-forge" -] -``` - -The GHCR mirror also contains `bioconda` packages. You can search the [available -packages on Github](https://github.com/orgs/channel-mirrors/packages). - - -### PyPI configuration -To setup a certain number of defaults for the usage of PyPI registries. You can use the following configuration options: - -- `index-url`: The default index URL to use for PyPI packages. This will be added to a manifest file on a pixi init. -- `extra-index-urls`: A list of additional URLs to use for PyPI packages. This will be added to a manifest file on a pixi init. -- `keyring-provider`: Allows the use of the [keyring](https://pypi.org/project/keyring/) python package to store and retrieve credentials. - -```toml -[pypi-config] -# Main index url -index-url = "https://pypi.org/simple" -# list of additional urls -extra-index-urls = ["https://pypi.org/simple2"] -# can be "subprocess" or "disabled" -keyring-provider = "subprocess" -``` - -!!! Note "`index-url` and `extra-index-urls` are *not* globals" - Unlike pip, these settings, with the exception of `keyring-provider` will only modify the `pixi.toml`/`pyproject.toml` file and are not globally interpreted when not present in the manifest. - This is because we want to keep the manifest file as complete and reproducible as possible. diff --git a/docs/advanced/pyproject_toml.md b/docs/advanced/pyproject_toml.md index c1c0a99e8..487a915da 100644 --- a/docs/advanced/pyproject_toml.md +++ b/docs/advanced/pyproject_toml.md @@ -110,7 +110,7 @@ As pixi takes the conda dependencies over the pypi dependencies. ## Optional dependencies -If your python project includes groups of optional dependencies, pixi will automatically interpret them as [pixi features](../reference/configuration.md#the-feature-table) of the same name with the associated `pypi-dependencies`. +If your python project includes groups of optional dependencies, pixi will automatically interpret them as [pixi features](../reference/project_configuration.md#the-feature-table) of the same name with the associated `pypi-dependencies`. You can add them to pixi environments manually, or use `pixi init` to setup the project, which will create one environment per feature. Self-references to other groups of optional dependencies are also handled. diff --git a/docs/features/advanced_tasks.md b/docs/features/advanced_tasks.md index 65815f535..9dd0030de 100644 --- a/docs/features/advanced_tasks.md +++ b/docs/features/advanced_tasks.md @@ -125,7 +125,7 @@ To add a task to run the `bar.py` file, use: pixi task add bar "python bar.py" --cwd scripts ``` -This will add the following line to [manifest file](../reference/configuration.md): +This will add the following line to [manifest file](../reference/project_configuration.md): ```toml title="pixi.toml" [tasks] diff --git a/docs/features/lockfile.md b/docs/features/lockfile.md index e4f442ae1..0defac749 100644 --- a/docs/features/lockfile.md +++ b/docs/features/lockfile.md @@ -104,8 +104,8 @@ Running the following commands will check and automatically update the lock file All the commands that support the interaction with the lock file also include some lock file usage options: -- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](../reference/configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). -- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](../reference/configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](../reference/project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](../reference/project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. !!! Note "Syncing the lock file with the manifest file" The lock file is always matched with the whole configuration in the manifest file. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ce276a16c..646637444 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -49,7 +49,7 @@ pixi init --import environment.yml ## `add` -Adds dependencies to the [manifest file](configuration.md). +Adds dependencies to the [manifest file](project_configuration.md). It will only add if the package with its version constraint is able to work with rest of the dependencies in the project. [More info](../features/multi_platform_configuration.md) on multi-platform configuration. @@ -65,12 +65,12 @@ These dependencies will be read by pixi as if they had been added to the pixi `p ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--host`: Specifies a host dependency, important for building a package. - `--build`: Specifies a build dependency, important for building a package. - `--pypi`: Specifies a PyPI dependency, not a conda package. Parses dependencies as [PEP508](https://peps.python.org/pep-0508/) requirements, supporting extras and versions. - See [configuration](configuration.md) for details. + See [configuration](project_configuration.md) for details. - `--no-install`: Don't install the package to the environment, only add the package to the lock-file. - `--no-lock file-update`: Don't update the lock-file, implies the `--no-install` flag. - `--platform (-p)`: The platform for which the dependency should be added. (Allowed to be used more than once) @@ -95,8 +95,8 @@ pixi add --feature featurex numpy ## `install` -Installs an environment based on the [manifest file](configuration.md). -If there is no `pixi.lock` file or it is not up-to-date with the [manifest file](configuration.md), it will (re-)generate the lock file. +Installs an environment based on the [manifest file](project_configuration.md). +If there is no `pixi.lock` file or it is not up-to-date with the [manifest file](project_configuration.md), it will (re-)generate the lock file. `pixi install` only installs one environment at a time, if you have multiple environments you can select the right one with the `--environment` flag. If you don't provide an environment, the `default` environment will be installed. @@ -106,9 +106,9 @@ As all commands interacting with the environment will first run the `install` co E.g. `pixi run`, `pixi shell`, `pixi shell-hook`, `pixi add`, `pixi remove` to name a few. ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. -- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). -- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. +- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--environment (-e)`: The environment to install, if none are provided the default environment will be used. ```shell @@ -136,7 +136,7 @@ This command will allow you to update the lock file directly, without manually d The `run` commands first checks if the environment is ready to use. When you didn't run `pixi install` the run command will do that for you. -The custom tasks defined in the [manifest file](configuration.md) are also available through the run command. +The custom tasks defined in the [manifest file](project_configuration.md) are also available through the run command. You cannot run `pixi run source setup.bash` as `source` is not available in the `deno_task_shell` commandos and not an executable. @@ -146,9 +146,9 @@ You cannot run `pixi run source setup.bash` as `source` is not available in the ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. -- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). -- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. +- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--environment (-e)`: The environment to run the task in, if none are provided the default environment will be used or a selector will be given to select the right environment. ```shell @@ -196,7 +196,7 @@ pixi run --environment cuda python ## `remove` -Removes dependencies from the [manifest file](configuration.md). +Removes dependencies from the [manifest file](project_configuration.md). If the project manifest is a `pyproject.toml`, removing a pypi dependency with the `--pypi` flag will remove it from either - the native pyproject `project.dependencies` array or the native `project.optional-dependencies` table (if a feature is specified) @@ -208,7 +208,7 @@ If the project manifest is a `pyproject.toml`, removing a pypi dependency with t ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--host`: Specifies a host dependency, important for building a package. - `--build`: Specifies a build dependency, important for building a package. - `--pypi`: Specifies a PyPI dependency, not a conda package. @@ -236,11 +236,11 @@ If you want to make a shorthand for a specific command you can add a task for it ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. ### `task add` -Add a task to the [manifest file](configuration.md), use `--depends-on` to add tasks you want to run before this task, e.g. build before an execute task. +Add a task to the [manifest file](project_configuration.md), use `--depends-on` to add tasks you want to run before this task, e.g. build before an execute task. ##### Arguments @@ -265,7 +265,7 @@ pixi task add build-osx "METAL=1 cargo build" --platform osx-64 pixi task add train python train.py --feature cuda ``` -This adds the following to the [manifest file](configuration.md): +This adds the following to the [manifest file](project_configuration.md): ```toml [tasks] @@ -290,7 +290,7 @@ pixi run test --test test1 ### `task remove` -Remove the task from the [manifest file](configuration.md) +Remove the task from the [manifest file](project_configuration.md) ##### Arguments @@ -351,10 +351,10 @@ List project's packages. Highlighted packages are explicit dependencies. - `--json`: Whether to output in json format. - `--json-pretty`: Whether to output in pretty json format - `--sort-by `: Sorting strategy [default: name] [possible values: size, name, type] -- `--manifest-path `: The path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: The path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--environment (-e)`: The environment's packages to list, if non is provided the default environment's packages will be listed. -- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). -- `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`) ```shell @@ -368,7 +368,7 @@ pixi list --locked pixi list --no-install ``` -Output will look like this, where `python` will be green as it is the package that was explicitly added to the [manifest file](configuration.md): +Output will look like this, where `python` will be green as it is the package that was explicitly added to the [manifest file](project_configuration.md): ```shell ➜ pixi list @@ -410,10 +410,10 @@ The package tree can also be inverted (`-i`), to see which packages require a sp - `--invert (-i)`: Invert the dependency tree, that is given a `REGEX` pattern that matches some packages, show all the packages that depend on those. - `--platform (-p)`: The platform to list packages for. Defaults to the current platform -- `--manifest-path `: The path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: The path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--environment (-e)`: The environment's packages to list, if non is provided the default environment's packages will be listed. -- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). -- `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`) ```shell @@ -427,7 +427,7 @@ pixi tree --platform win-64 !!! warning Use `-v` to show which `pypi` packages are not yet parsed correctly. The `extras` and `markers` parsing is still under development. -Output will look like this, where direct packages in the [manifest file](configuration.md) will be green. +Output will look like this, where direct packages in the [manifest file](project_configuration.md) will be green. Once a package has been displayed once, the tree won't continue to recurse through its dependencies (compare the first time `python` appears, vs the rest), and it will instead be marked with a star `(*)`. Version numbers are colored by the package type, yellow for Conda packages and blue for PyPI. @@ -521,9 +521,9 @@ To exit the pixi shell, simply run `exit`. ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. -- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). -- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. +- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--environment (-e)`: The environment to activate the shell in, if none are provided the default environment will be used or a selector will be given to select the right environment. ```shell @@ -547,9 +547,9 @@ This command prints the activation script of an environment. - `--shell (-s)`: The shell for which the activation script should be printed. Defaults to the current shell. Currently supported variants: [`bash`, `zsh`, `xonsh`, `cmd`, `powershell`, `fish`, `nushell`] -- `--manifest-path`: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. -- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). -- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--manifest-path`: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. +- `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--environment (-e)`: The environment to activate, if none are provided the default environment will be used or a selector will be given to select the right environment. - `--json`: Print all environment variables that are exported by running the activation script as JSON. When specifying this option, `--shell` is ignored. @@ -583,7 +583,7 @@ Search a package, output will list the latest version of the package. ###### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--channel (-c)`: specify a channel that the project uses. Defaults to `conda-forge`. (Allowed to be used more than once) - `--limit (-l)`: optionally limit the number of search results - `--platform (-p)`: specify a platform that you want to search for. (default: current platform) @@ -619,7 +619,7 @@ More information [here](../advanced/explain_info_command.md). ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--extended`: extend the information with more slow queries to the system, like directory sizes. - `--json`: Get a machine-readable version of the information as output. @@ -811,7 +811,7 @@ This subcommand allows you to modify the project configuration through the comma ##### Options -- `--manifest-path `: the path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. ### `project channel add` diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md new file mode 100644 index 000000000..edfaad202 --- /dev/null +++ b/docs/reference/pixi_configuration.md @@ -0,0 +1,209 @@ +# The configuration of pixi itself + +Apart from the [project specific configuration](../reference/project_configuration.md) pixi supports configuration options which are not required for the project to work but are local to the machine. +The configuration is loaded in the following order: + + +=== "Linux" + + | **Priority** | **Location** | **Comments** | + |--------------|------------------------------------------------------------------------|------------------------------------------------------------------------------------| + | 1 | `/etc/pixi/config.toml` | System-wide configuration | + | 2 | `$XDG_CONFIG_HOME/pixi/config.toml` | XDG compliant user-specific configuration | + | 3 | `$HOME/.config/pixi/config.toml` | User-specific configuration | + | 4 | `$PIXI_HOME/config.toml` | Global configuration in the user home directory. `PIXI_HOME` defaults to `~/.pixi` | + | 5 | `your_project/.pixi/config.toml` | Project-specific configuration | + | 6 | Command line arguments (`--tls-no-verify`, `--change-ps1=false`, etc.) | Configuration via command line arguments | + +=== "macOS" + + | **Priority** | **Location** | **Comments** | + |--------------|------------------------------------------------------------------------|------------------------------------------------------------------------------------| + | 1 | `/etc/pixi/config.toml` | System-wide configuration | + | 2 | `$XDG_CONFIG_HOME/pixi/config.toml` | XDG compliant user-specific configuration | + | 3 | `$HOME/Library/Application Support/pixi/config.toml` | User-specific configuration | + | 4 | `$PIXI_HOME/config.toml` | Global configuration in the user home directory. `PIXI_HOME` defaults to `~/.pixi` | + | 5 | `your_project/.pixi/config.toml` | Project-specific configuration | + | 6 | Command line arguments (`--tls-no-verify`, `--change-ps1=false`, etc.) | Configuration via command line arguments | + +=== "Windows" + + | **Priority** | **Location** | **Comments** | + |--------------|------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| + | 1 | `C:\ProgramData\pixi\config.toml` | System-wide configuration | + | 2 | `%APPDATA%\pixi\config.toml` | User-specific configuration | + | 3 | `$PIXI_HOME\config.toml` | Global configuration in the user home directory. `PIXI_HOME` defaults to `%USERPROFILE%/.pixi` | + | 4 | `your_project\.pixi\config.toml` | Project-specific configuration | + | 5 | Command line arguments (`--tls-no-verify`, `--change-ps1=false`, etc.) | Configuration via command line arguments | + +!!! note + The highest priority wins. If a configuration file is found in a higher priority location, the lower priority locations are overwritten. + + +!!! note + To find the locations where `pixi` looks for configuration files, run + `pixi` with `-v` or `--verbose`. + +## Reference + +??? info "Casing In Configuration" + In versions of pixi `0.20.1` and older the global configuration used snake_case + we've changed to `kebab-case` for consistency with the rest of the configuration. + But we still support the old `snake_case` configuration, for older configuration options. + These are: + + - `default_channels` + - `change_ps1` + - `tls_no_verify` + - `authentication_override_file` + - `mirrors` and sub-options + - `repodata-config` and sub-options + +The following reference describes all available configuration options. + +### `default-channels` + +The default channels to select when running `pixi init` or `pixi global install`. +This defaults to only conda-forge. +```toml title="config.toml" +default-channels = ["conda-forge"] +``` + +### `change-ps1` + +When set to false, the `(pixi)` prefix in the shell prompt is removed. +This applies to the `pixi shell` subcommand. +You can override this from the CLI with `--change-ps1`. + +```toml title="config.toml" +change-ps1 = true +``` + +### `tls-no-verify` +When set to true, the TLS certificates are not verified. + +!!! warning + + This is a security risk and should only be used for testing purposes or internal networks. + +You can override this from the CLI with `--tls-no-verify`. + +```toml title="config.toml" +tls-no-verify = false +``` + +### `authentication-override-file` +Override from where the authentication information is loaded. +Usually, we try to use the keyring to load authentication data from, and only use a JSON +file as a fallback. This option allows you to force the use of a JSON file. +Read more in the authentication section. +```toml title="config.toml" +authentication-override-file = "/path/to/your/override.json" +``` + +### `target-environment-directory` +The directory where pixi stores the environments, what would normally be placed in the `.pixi` folder in a project's root. +This is not recommended to change, but can be useful in some cases. + +- Forcing the installation on a specific filesystem/drive +- Network hosted projects, but local environments +- Let the system-administrator have more control over all environments on a system. + +```toml title="config.toml" +target-environment-diretory = "/opt/pixi/envs" +``` + +### `mirrors` +Configuration for conda channel-mirrors, more info [below](#mirror-configuration). + +```toml title="config.toml" +[mirrors] +# redirect all requests for conda-forge to the prefix.dev mirror +"https://conda.anaconda.org/conda-forge" = [ + "https://prefix.dev/conda-forge" +] + +# redirect all requests for bioconda to one of the three listed mirrors +# Note: for repodata we try the first mirror first. +"https://conda.anaconda.org/bioconda" = [ + "https://conda.anaconda.org/bioconda", + # OCI registries are also supported + "oci://ghcr.io/channel-mirrors/bioconda", + "https://prefix.dev/bioconda", +] +``` + +### `repodata-config` +Configuration for repodata fetching. +```toml title="config.toml" +[repodata-config] +# disable fetching of jlap, bz2 or zstd repodata files. +# This should only be used for specific old versions of artifactory and other non-compliant +# servers. +disable-jlap = true # don't try to download repodata.jlap +disable-bzip2 = true # don't try to download repodata.json.bz2 +disable-zstd = true # don't try to download repodata.json.zst +``` + +### `pypi-config` +To setup a certain number of defaults for the usage of PyPI registries. You can use the following configuration options: + +- `index-url`: The default index URL to use for PyPI packages. This will be added to a manifest file on a `pixi init`. +- `extra-index-urls`: A list of additional URLs to use for PyPI packages. This will be added to a manifest file on a `pixi init`. +- `keyring-provider`: Allows the use of the [keyring](https://pypi.org/project/keyring/) python package to store and retrieve credentials. + +```toml title="config.toml" +[pypi-config] +# Main index url +index-url = "https://pypi.org/simple" +# list of additional urls +extra-index-urls = ["https://pypi.org/simple2"] +# can be "subprocess" or "disabled" +keyring-provider = "subprocess" +``` + +!!! Note "`index-url` and `extra-index-urls` are *not* globals" + Unlike pip, these settings, with the exception of `keyring-provider` will only modify the `pixi.toml`/`pyproject.toml` file and are not globally interpreted when not present in the manifest. + This is because we want to keep the manifest file as complete and reproducible as possible. + +## Mirror configuration + +You can configure mirrors for conda channels. We expect that mirrors are exact +copies of the original channel. The implementation will look for the mirror key +(a URL) in the `mirrors` section of the configuration file and replace the +original URL with the mirror URL. + +To also include the original URL, you have to repeat it in the list of mirrors. + +The mirrors are prioritized based on the order of the list. We attempt to fetch +the repodata (the most important file) from the first mirror in the list. The +repodata contains all the SHA256 hashes of the individual packages, so it is +important to get this file from a trusted source. + +You can also specify mirrors for an entire "host", e.g. + +```toml title="config.toml" +[mirrors] +"https://conda.anaconda.org" = [ + "https://prefix.dev/" +] +``` + +This will forward all request to channels on anaconda.org to prefix.dev. +Channels that are not currently mirrored on prefix.dev will fail in the above example. + +### OCI Mirrors + +You can also specify mirrors on the OCI registry. There is a public mirror on +the Github container registry (ghcr.io) that is maintained by the conda-forge +team. You can use it like this: + +```toml title="config.toml" +[mirrors] +"https://conda.anaconda.org/conda-forge" = [ + "oci://ghcr.io/channel-mirrors/conda-forge" +] +``` + +The GHCR mirror also contains `bioconda` packages. You can search the [available +packages on Github](https://github.com/orgs/channel-mirrors/packages). diff --git a/docs/reference/configuration.md b/docs/reference/project_configuration.md similarity index 99% rename from docs/reference/configuration.md rename to docs/reference/project_configuration.md index 38c5aeb57..4ed366659 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/project_configuration.md @@ -692,4 +692,4 @@ When an environment comprises several features (including the default feature): ## Global configuration -The global configuration options are documented in the [global configuration](../advanced/global_configuration.md) section. +The global configuration options are documented in the [global configuration](../reference/pixi_configuration.md) section. diff --git a/docs/tutorials/python.md b/docs/tutorials/python.md index 435f661bc..91d4f1eac 100644 --- a/docs/tutorials/python.md +++ b/docs/tutorials/python.md @@ -142,7 +142,7 @@ Which results in the following fields added to the `pyproject.toml`: test = ["pytest"] ``` -After we have added the optional dependencies to the `pyproject.toml`, pixi automatically creates a [`feature`](../reference/configuration.md/#the-feature-and-environments-tables), which can contain a collection of `dependencies`, `tasks`, `channels`, and more. +After we have added the optional dependencies to the `pyproject.toml`, pixi automatically creates a [`feature`](../reference/project_configuration.md/#the-feature-and-environments-tables), which can contain a collection of `dependencies`, `tasks`, `channels`, and more. Sometimes there are packages that aren't available on conda channels but are published on PyPI. We can add these as well, which pixi will solve together with the default dependencies. diff --git a/docs/tutorials/ros2.md b/docs/tutorials/ros2.md index 0932ddc1b..1d31eff29 100644 --- a/docs/tutorials/ros2.md +++ b/docs/tutorials/ros2.md @@ -171,7 +171,7 @@ pixi run hello - You can add [`depends-on`](../features/advanced_tasks.md#depends-on) to the tasks to create a task chain. - You can add [`cwd`](../features/advanced_tasks.md#working-directory) to the tasks to run the task in a different directory from the root of the project. - You can add [`inputs` and `outputs`](../features/advanced_tasks.md#caching) to the tasks to create a task that only runs when the inputs are changed. - - You can use the [`target`](../reference/configuration.md#the-target-table) syntax to run specific tasks on specific machines. + - You can use the [`target`](../reference/project_configuration.md#the-target-table) syntax to run specific tasks on specific machines. ```toml [tasks] diff --git a/mkdocs.yml b/mkdocs.yml index 1ebcb4b08..1e8a4ff6f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -122,10 +122,10 @@ nav: - Info Command: advanced/explain_info_command.md - Channel Logic: advanced/channel_priority.md - GitHub Actions: advanced/github_actions.md - - Global Configuration: advanced/global_configuration.md - Pyproject.toml: advanced/pyproject_toml.md - Reference: - - Project Configuration: reference/configuration.md + - Project Configuration: reference/project_configuration.md + - Pixi Configuration: reference/pixi_configuration.md - CLI: reference/cli.md - Misc: - Pixi vision: vision.md @@ -144,7 +144,8 @@ plugins: "design_proposals/multi_environment_proposal.md": "features/multi_environment.md" "advanced/multi_platform_configuration.md": "features/multi_platform_configuration.md" "cli.md": "reference/cli.md" - "configuration.md": "reference/configuration.md" + "configuration.md": "reference/project_configuration.md" + "advanced/global_configuration.md": "reference/pixi_configuration.md" - search - social diff --git a/src/config.rs b/src/config.rs index 4d004bd9a..b4648d844 100644 --- a/src/config.rs +++ b/src/config.rs @@ -216,7 +216,7 @@ pub struct Config { /// The location of the environments build by pixi #[serde(default)] #[serde( - alias = "target-environments-directory", + rename = "target-environments-directory", skip_serializing_if = "Option::is_none" )] pub target_environments_directory: Option, From 350151b51f6e4517f71601b7fcf77047585fd4d9 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 13:44:35 +0200 Subject: [PATCH 06/34] test: fix for windows --- tests/install_tests.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index ceac39717..19595d78b 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -185,11 +185,18 @@ async fn install_locked_with_config() { )); // Verify that the folders are present in the target directory using a task. + #[cfg(not(target_os = "windows"))] pixi.tasks() .add("which_python".into(), None, FeatureName::Default) .with_commands(["which python"]) .execute() .unwrap(); + #[cfg(target_os = "windows")] + pixi.tasks() + .add("which_python".into(), None, FeatureName::Default) + .with_commands(["where python"]) + .execute() + .unwrap(); let result = pixi .run(Args { @@ -200,7 +207,9 @@ async fn install_locked_with_config() { .await .unwrap(); assert_eq!(result.exit_code, 0); - assert!(result.stdout.contains(target_dir.to_str().unwrap())); + // Check for correct path in most important path + let line = result.stdout.lines().next().unwrap(); + assert!(line.contains(target_dir.to_str().unwrap())); } /// Test `pixi install/run --frozen` functionality From 2f19e71dd7a9805cb43f05652616b9986da37f56 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 14:36:40 +0200 Subject: [PATCH 07/34] misc: Add more warnings for the user --- src/cli/install.rs | 23 ++++++++++++++++++----- src/project/mod.rs | 27 ++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/cli/install.rs b/src/cli/install.rs index 16757ff69..014804519 100644 --- a/src/cli/install.rs +++ b/src/cli/install.rs @@ -36,11 +36,24 @@ pub async fn execute(args: Args) -> miette::Result<()> { .await?; // Emit success - eprintln!( - "{}Project in {} is ready to use!", - console::style(console::Emoji("✔ ", "")).green(), - project.root().display() - ); + if project.config().target_environments_directory.is_some() { + eprintln!( + "{}Project environment is installed in '{}' and is ready to use!", + console::style(console::Emoji("✔ ", "")).green(), + project + .config() + .target_environments_directory + .as_ref() + .unwrap() + .display() + ); + } else { + eprintln!( + "{}Project in {} is ready to use!", + console::style(console::Emoji("✔ ", "")).green(), + project.root().display() + ); + } Project::warn_on_discovered_from_env(args.manifest_path.as_deref()); Ok(()) } diff --git a/src/project/mod.rs b/src/project/mod.rs index 76e6c8988..6128491b3 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -43,8 +43,11 @@ use self::{ }; pub use dependencies::{CondaDependencies, PyPiDependencies}; pub use environment::Environment; +use once_cell::sync::OnceCell; pub use solve_group::SolveGroup; +static CUSTOM_TARGET_DIR_WARN: OnceCell<()> = OnceCell::new(); + /// The dependency types we support #[derive(Debug, Copy, Clone)] pub enum DependencyType { @@ -285,12 +288,26 @@ impl Project { /// Returns the pixi directory pub fn pixi_dir(&self) -> PathBuf { + let default_pixi_dir = self.root.join(consts::PIXI_DIR); + // Custom root directory for target environments if set in configuration. if let Some(custom_root) = self.config().target_environments_directory() { - tracing::info!( - "Using custom target directory for environments: {}", - custom_root.display() - ); + let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { + tracing::info!( + "Using custom target directory for environments: {}", + custom_root.display() + ); + // Warn user if environments are found in the default directory + if default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() { + tracing::warn!( + "Environments found in '{}', this will be ignored in favor of custom target directory '{}'\n\ + \t\tIt's advised to remove the environments from the default directory to avoid confusion.", + default_pixi_dir.display(), + custom_root.display() + ); + } + }); + return custom_root.join(format!( "{}-{}", self.name(), @@ -298,7 +315,7 @@ impl Project { )); } tracing::debug!("Using default root directory for target environments"); - self.root.join(consts::PIXI_DIR) + default_pixi_dir } /// Returns the environment directory From 6f68a9cc4300a709e167687311c9ea21749a15a0 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 15:29:21 +0200 Subject: [PATCH 08/34] feat: add symlink if possible/needed --- src/project/mod.rs | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/project/mod.rs b/src/project/mod.rs index 6128491b3..70e719c3d 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -16,6 +16,7 @@ use reqwest_middleware::ClientWithMiddleware; use std::hash::Hash; use rattler_virtual_packages::VirtualPackage; +use std::os::unix::fs::symlink; use std::{ collections::{HashMap, HashSet}, env, @@ -292,27 +293,41 @@ impl Project { // Custom root directory for target environments if set in configuration. if let Some(custom_root) = self.config().target_environments_directory() { + let pixi_dir_name = custom_root.join(format!( + "{}-{}", + self.name(), + xxh3_64(self.root.to_string_lossy().as_bytes()) + )); let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { tracing::info!( "Using custom target directory for environments: {}", - custom_root.display() + pixi_dir_name.display() ); // Warn user if environments are found in the default directory - if default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() { - tracing::warn!( - "Environments found in '{}', this will be ignored in favor of custom target directory '{}'\n\ - \t\tIt's advised to remove the environments from the default directory to avoid confusion.", - default_pixi_dir.display(), - custom_root.display() - ); + if !default_pixi_dir.is_symlink() { + if default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() { + tracing::warn!( + "Environments found in '{}', this will be ignored in favor of custom target directory '{}'\n\ + \t\tIt's advised to remove the environments from the default directory to avoid confusion.", + default_pixi_dir.display(), + custom_root.display() + ); + } else { + // Create symlink from custom root to default pixi directory + if cfg!(unix) { + match symlink(pixi_dir_name.clone(), default_pixi_dir.clone()) { + Ok(_) => tracing::info!("Symlink created successfully from {} to {} folder", custom_root.display(), default_pixi_dir.display()), + Err(e) => println!("Failed to create symlink: {}", e), + } + } else { + tracing::info!("Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory."); + } + } } + }); - return custom_root.join(format!( - "{}-{}", - self.name(), - xxh3_64(self.root.to_string_lossy().as_bytes()) - )); + return pixi_dir_name; } tracing::debug!("Using default root directory for target environments"); default_pixi_dir From 0db24eacef9f3aa3d1bd8ae2e8c03711c098ab15 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 16:04:27 +0200 Subject: [PATCH 09/34] fix: windows build --- src/project/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/project/mod.rs b/src/project/mod.rs index 70e719c3d..9f5932f67 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -16,7 +16,9 @@ use reqwest_middleware::ClientWithMiddleware; use std::hash::Hash; use rattler_virtual_packages::VirtualPackage; +#[cfg(unix)] use std::os::unix::fs::symlink; + use std::{ collections::{HashMap, HashSet}, env, From 20cb5e719b32f32b945ae34c4e20d9f7584a4a33 Mon Sep 17 00:00:00 2001 From: "Ruben Arts(win)" Date: Tue, 14 May 2024 16:38:12 +0200 Subject: [PATCH 10/34] fix: compile on win --- src/project/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/project/mod.rs b/src/project/mod.rs index 9f5932f67..af5b5a9f0 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -16,7 +16,7 @@ use reqwest_middleware::ClientWithMiddleware; use std::hash::Hash; use rattler_virtual_packages::VirtualPackage; -#[cfg(unix)] +#[cfg(not(windows))] use std::os::unix::fs::symlink; use std::{ @@ -316,14 +316,14 @@ impl Project { ); } else { // Create symlink from custom root to default pixi directory - if cfg!(unix) { - match symlink(pixi_dir_name.clone(), default_pixi_dir.clone()) { + #[cfg(not(windows))] + match symlink(pixi_dir_name.clone(), default_pixi_dir.clone()) { Ok(_) => tracing::info!("Symlink created successfully from {} to {} folder", custom_root.display(), default_pixi_dir.display()), Err(e) => println!("Failed to create symlink: {}", e), - } - } else { - tracing::info!("Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory."); } + + #[cfg(windows)] + tracing::info!("Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory."); } } From d5ca37b136099041279a5ce7058d25f1da10c79a Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 14 May 2024 16:43:41 +0200 Subject: [PATCH 11/34] fix: docs build --- docs/advanced/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced/authentication.md b/docs/advanced/authentication.md index e3c71990b..79e41d627 100644 --- a/docs/advanced/authentication.md +++ b/docs/advanced/authentication.md @@ -191,7 +191,7 @@ gcloud artifacts print-settings python --project= --repository= Date: Wed, 15 May 2024 09:39:43 +0200 Subject: [PATCH 12/34] fix: test on win --- src/config.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/config.rs b/src/config.rs index b4648d844..619b39871 100644 --- a/src/config.rs +++ b/src/config.rs @@ -453,12 +453,11 @@ mod tests { #[test] fn test_config_parse() { let toml = format!( - r#" - default_channels = ["conda-forge"] - tls_no_verify = true - target-environments-directory = "{}" + r#"default-channels = ["conda-forge"] +tls-no-verify = true +target-environments-directory = "{}" "#, - env!("CARGO_MANIFEST_DIR") + env!("CARGO_MANIFEST_DIR").replace("\\", "\\\\").as_str() ); let config = Config::from_toml(toml.as_str(), &PathBuf::from("")).unwrap(); assert_eq!(config.default_channels, vec!["conda-forge"]); From ee87067de74eabfc8d059fd9a1cc13e19db91e27 Mon Sep 17 00:00:00 2001 From: "Ruben Arts(win)" Date: Wed, 15 May 2024 10:25:44 +0200 Subject: [PATCH 13/34] misc: dbg ci --- tests/install_tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index 19595d78b..726af8e33 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -124,7 +124,7 @@ async fn test_incremental_lock_file() { /// Test the `pixi install --locked` functionality. #[tokio::test] #[serial] -#[cfg_attr(not(feature = "slow_integration_tests"), ignore)] +// #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn install_locked_with_config() { let pixi = PixiControl::new().unwrap(); pixi.init().await.unwrap(); @@ -209,6 +209,7 @@ async fn install_locked_with_config() { assert_eq!(result.exit_code, 0); // Check for correct path in most important path let line = result.stdout.lines().next().unwrap(); + dbg!(line); assert!(line.contains(target_dir.to_str().unwrap())); } From 5dfc35e5dc88012604b818bd592aa916cb5474c1 Mon Sep 17 00:00:00 2001 From: "Ruben Arts(win)" Date: Wed, 15 May 2024 11:17:34 +0200 Subject: [PATCH 14/34] fix: test on win, canonicalize path --- tests/install_tests.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index 726af8e33..8552caf94 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -2,7 +2,7 @@ mod common; use std::fs::{create_dir_all, File}; use std::io::Write; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use crate::common::builders::string_from_iter; @@ -207,10 +207,12 @@ async fn install_locked_with_config() { .await .unwrap(); assert_eq!(result.exit_code, 0); + // Check for correct path in most important path let line = result.stdout.lines().next().unwrap(); - dbg!(line); - assert!(line.contains(target_dir.to_str().unwrap())); + let target_dir_canonical = target_dir.canonicalize().unwrap(); + let line_path = PathBuf::from(line).canonicalize().unwrap(); + assert!(line_path.starts_with(&target_dir_canonical)); } /// Test `pixi install/run --frozen` functionality From 602952673d44d3bfc59616004c42dbe7d4dcfb7d Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 15 May 2024 11:36:48 +0200 Subject: [PATCH 15/34] Update tests/install_tests.rs --- tests/install_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index 8552caf94..b7971f057 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -134,7 +134,6 @@ async fn install_locked_with_config() { let target_dir = pixi.project_path().join("target"); config.target_environments_directory = Some(target_dir.clone()); create_dir_all(target_dir.clone()).unwrap(); - dbg!(config.clone().target_environments_directory); let config_path = pixi.project().unwrap().pixi_dir().join("config.toml"); create_dir_all(config_path.parent().unwrap()).unwrap(); From 88c02d06b7f607051c16db5c5ac1f3389320e3b2 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 15 May 2024 11:36:53 +0200 Subject: [PATCH 16/34] Update tests/install_tests.rs --- tests/install_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index b7971f057..81a071bae 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -124,7 +124,7 @@ async fn test_incremental_lock_file() { /// Test the `pixi install --locked` functionality. #[tokio::test] #[serial] -// #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] +#[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn install_locked_with_config() { let pixi = PixiControl::new().unwrap(); pixi.init().await.unwrap(); From 2ea5f14509b5b18dbe361f3cbc10ec61338032ac Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 10:03:51 +0200 Subject: [PATCH 17/34] docs: clearify the environments affected by target env dir --- docs/reference/pixi_configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index edfaad202..60086a4f5 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -102,7 +102,8 @@ authentication-override-file = "/path/to/your/override.json" ``` ### `target-environment-directory` -The directory where pixi stores the environments, what would normally be placed in the `.pixi` folder in a project's root. +The directory where pixi stores the project environments, what would normally be placed in the `.pixi` folder in a project's root. +It doesn't affect the environments build with `pixi global` as they listen to the `PIXI_HOME` environment variable. This is not recommended to change, but can be useful in some cases. - Forcing the installation on a specific filesystem/drive From cca6ac0c3ad65401ed74bddfc205334c42e491dc Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 10:37:32 +0200 Subject: [PATCH 18/34] Update docs/reference/pixi_configuration.md Co-authored-by: Bas Zalmstra --- docs/reference/pixi_configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index 60086a4f5..5df6c0929 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -103,7 +103,8 @@ authentication-override-file = "/path/to/your/override.json" ### `target-environment-directory` The directory where pixi stores the project environments, what would normally be placed in the `.pixi` folder in a project's root. -It doesn't affect the environments build with `pixi global` as they listen to the `PIXI_HOME` environment variable. +It doesn't affect the environments built for `pixi global`. +The location of environments created for a `pixi global` installation can be controlled using the `PIXI_HOME` environment variable. This is not recommended to change, but can be useful in some cases. - Forcing the installation on a specific filesystem/drive From 4dec0e96a091bc08f71e674f64248217416ee7de Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 10:37:44 +0200 Subject: [PATCH 19/34] Update docs/reference/pixi_configuration.md Co-authored-by: Bas Zalmstra --- docs/reference/pixi_configuration.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index 5df6c0929..b05e9cad3 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -105,11 +105,14 @@ authentication-override-file = "/path/to/your/override.json" The directory where pixi stores the project environments, what would normally be placed in the `.pixi` folder in a project's root. It doesn't affect the environments built for `pixi global`. The location of environments created for a `pixi global` installation can be controlled using the `PIXI_HOME` environment variable. -This is not recommended to change, but can be useful in some cases. +!!! warning + We recommend against using this because any environment created for a project is no longer placed in the same folder as the project. + This creates a disconnect between the project and its environments and manual cleanup of the environments is required when deleting the project. + However, in some cases, this option can still be very useful, for instance to: -- Forcing the installation on a specific filesystem/drive -- Network hosted projects, but local environments -- Let the system-administrator have more control over all environments on a system. + - force the installation on a specific filesystem/drive. + - install environments locally but keep the project on a network drive. + - let a system-administrator have more control over all environments on a system. ```toml title="config.toml" target-environment-diretory = "/opt/pixi/envs" From 0dafb10f02d2c41746ef770339b8381dc41ae7f5 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 10:37:52 +0200 Subject: [PATCH 20/34] Update src/config.rs Co-authored-by: Bas Zalmstra --- src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 619b39871..83d7fc52a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -441,8 +441,8 @@ impl Config { } /// Retrieve the value for the target_environments_directory field. - pub fn target_environments_directory(&self) -> Option { - self.target_environments_directory.clone() + pub fn target_environments_directory(&self) -> Option<&Path> { + self.target_environments_directory.as_deref() } } From 3a0ccb63fee29baf7b2f8728c7ce2c2ea3897818 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 10:38:09 +0200 Subject: [PATCH 21/34] Update docs/reference/pixi_configuration.md Co-authored-by: Bas Zalmstra --- docs/reference/pixi_configuration.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index b05e9cad3..7d188feec 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -68,7 +68,9 @@ This defaults to only conda-forge. ```toml title="config.toml" default-channels = ["conda-forge"] ``` - +!!! note + The `default-channels` are only used when initializing a new project. Once initialized the `channels` are used from the project manifest. + ### `change-ps1` When set to false, the `(pixi)` prefix in the shell prompt is removed. From 4a2fa40995060f2974ee23e4e96001e65a7916d8 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 10:38:22 +0200 Subject: [PATCH 22/34] Update docs/reference/pixi_configuration.md Co-authored-by: Bas Zalmstra --- docs/reference/pixi_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index 7d188feec..9e56239bd 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -37,7 +37,7 @@ The configuration is loaded in the following order: | 5 | Command line arguments (`--tls-no-verify`, `--change-ps1=false`, etc.) | Configuration via command line arguments | !!! note - The highest priority wins. If a configuration file is found in a higher priority location, the lower priority locations are overwritten. + The highest priority wins. If a configuration file is found in a higher priority location, the values from the configuration read from lower priority locations are overwritten. !!! note From 5a83bb0bd534e1b496b472aa89d1f429f1db577d Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 11:18:50 +0200 Subject: [PATCH 23/34] misc: cleanup and improve code --- docs/reference/pixi_configuration.md | 6 +++--- src/config.rs | 24 +++++++++++------------- src/project/mod.rs | 5 ++++- tests/install_tests.rs | 16 ++++++++-------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index 9e56239bd..75c7c855b 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -70,7 +70,7 @@ default-channels = ["conda-forge"] ``` !!! note The `default-channels` are only used when initializing a new project. Once initialized the `channels` are used from the project manifest. - + ### `change-ps1` When set to false, the `(pixi)` prefix in the shell prompt is removed. @@ -105,10 +105,10 @@ authentication-override-file = "/path/to/your/override.json" ### `target-environment-directory` The directory where pixi stores the project environments, what would normally be placed in the `.pixi` folder in a project's root. -It doesn't affect the environments built for `pixi global`. +It doesn't affect the environments built for `pixi global`. The location of environments created for a `pixi global` installation can be controlled using the `PIXI_HOME` environment variable. !!! warning - We recommend against using this because any environment created for a project is no longer placed in the same folder as the project. + We recommend against using this because any environment created for a project is no longer placed in the same folder as the project. This creates a disconnect between the project and its environments and manual cleanup of the environments is required when deleting the project. However, in some cases, this option can still be very useful, for instance to: diff --git a/src/config.rs b/src/config.rs index 83d7fc52a..d89095d15 100644 --- a/src/config.rs +++ b/src/config.rs @@ -173,52 +173,49 @@ impl PyPIConfig { } #[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] pub struct Config { #[serde(default)] - #[serde(alias = "default-channels")] // BREAK: rename instead of alias + #[serde(alias = "default_channels")] // BREAK(0.22.0): remove alias pub default_channels: Vec, /// If set to true, pixi will set the PS1 environment variable to a custom value. #[serde(default)] - #[serde(alias = "change-ps1")] // BREAK: rename instead of alias + #[serde(alias = "change_ps1")] // BREAK(0.22.0): remove alias change_ps1: Option, /// Path to the file containing the authentication token. #[serde(default)] - #[serde(alias = "authentication-override-file")] // BREAK: rename instead of alias + #[serde(alias = "authentication_override_file")] // BREAK(0.22.0): remove alias authentication_override_file: Option, /// If set to true, pixi will not verify the TLS certificate of the server. #[serde(default)] - #[serde(alias = "tls-no-verify")] // BREAK: rename instead of alias + #[serde(alias = "tls_no_verify")] // BREAK(0.22.0): remove alias tls_no_verify: Option, #[serde(default)] mirrors: HashMap>, #[serde(skip)] - #[serde(alias = "loaded-from")] // BREAK: rename instead of alias + #[serde(alias = "loaded_from")] // BREAK(0.22.0): remove alias pub loaded_from: Vec, #[serde(skip, default = "default_channel_config")] - #[serde(alias = "channel-config")] // BREAK: rename instead of alias + #[serde(alias = "channel_config")] // BREAK(0.22.0): remove alias pub channel_config: ChannelConfig, /// Configuration for repodata fetching. - #[serde(alias = "repodata-config")] // BREAK: rename instead of alias + #[serde(alias = "repodata_config")] // BREAK(0.22.0): remove alias pub repodata_config: Option, /// Configuration for PyPI packages. #[serde(default)] - #[serde(rename = "pypi-config")] pub pypi_config: PyPIConfig, /// The location of the environments build by pixi #[serde(default)] - #[serde( - rename = "target-environments-directory", - skip_serializing_if = "Option::is_none" - )] + #[serde(skip_serializing_if = "Option::is_none")] pub target_environments_directory: Option, } @@ -278,7 +275,8 @@ impl Config { /// Validate the config file. pub fn validate(&self) -> miette::Result<()> { if let Some(target_env_dir) = self.target_environments_directory.clone() { - if !target_env_dir.exists() { + if !target_env_dir.is_absolute() || !target_env_dir.exists() { + // The path might exist, but we need it to be absolute because we don't canonicalize it. return Err(miette!("The `target-environments-directory` path does not exist: {:?}. It needs to be an absolute path to a directory.", target_env_dir)); } } diff --git a/src/project/mod.rs b/src/project/mod.rs index 3eacaca16..7bd8d009b 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -341,7 +341,10 @@ impl Project { return pixi_dir_name; } - tracing::debug!("Using default root directory for target environments"); + tracing::debug!( + "Using default root directory: `{}` for target environments.", + default_pixi_dir.display() + ); default_pixi_dir } diff --git a/tests/install_tests.rs b/tests/install_tests.rs index 81a071bae..ff7d6d6f7 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -183,17 +183,17 @@ async fn install_locked_with_config() { "python==3.9.0" )); + // Task command depends on the OS + let which_command = if cfg!(target_os = "windows") { + "where python" + } else { + "which python" + }; + // Verify that the folders are present in the target directory using a task. - #[cfg(not(target_os = "windows"))] - pixi.tasks() - .add("which_python".into(), None, FeatureName::Default) - .with_commands(["which python"]) - .execute() - .unwrap(); - #[cfg(target_os = "windows")] pixi.tasks() .add("which_python".into(), None, FeatureName::Default) - .with_commands(["where python"]) + .with_commands([which_command]) .execute() .unwrap(); From 35301fe152af92a1a62cfbe93ebd4ac5d0120a52 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 14:00:42 +0200 Subject: [PATCH 24/34] feat: write warning file to windows machines instead of the symlink --- src/project/mod.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/project/mod.rs b/src/project/mod.rs index 7bd8d009b..abffda2ff 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -319,21 +319,29 @@ impl Project { if !default_pixi_dir.is_symlink() { if default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() { tracing::warn!( - "Environments found in '{}', this will be ignored in favor of custom target directory '{}'\n\ - \t\tIt's advised to remove the environments from the default directory to avoid confusion.", + "Environments found in '{}', this will be ignored and the environment will be installed in the custom target directory '{}'\n\ + \t\tIt's advised to remove the {} folder from the default directory to avoid confusion{}.", default_pixi_dir.display(), - custom_root.display() + custom_root.display(), + consts::PIXI_DIR, + if cfg!(windows) { "" } else { " and a symlink to be made. Re-install if needed." } ); } else { + + if cfg!(windows) { + tracing::warn!("Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory."); + // Write file with warning to the default pixi directory + let warning_file = default_pixi_dir.join("README.txt"); + std::fs::write(&warning_file, format!("Environments are installed in a custom target directory: {}.\nSymlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory.", pixi_dir_name.display())).map_err(|e| println!("Failed to write warning file to {}: {}", warning_file.display(), e)).ok(); + } + // Create symlink from custom root to default pixi directory #[cfg(not(windows))] match symlink(pixi_dir_name.clone(), default_pixi_dir.clone()) { - Ok(_) => tracing::info!("Symlink created successfully from {} to {} folder", custom_root.display(), default_pixi_dir.display()), - Err(e) => println!("Failed to create symlink: {}", e), + Ok(_) => tracing::info!("Symlink created successfully from {} to {} folder", custom_root.display(), default_pixi_dir.display()), + Err(e) => println!("Failed to create symlink: {}", e), } - #[cfg(windows)] - tracing::info!("Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory."); } } From 64a9d7c90364a8aab6010283e45cf29e171d8cc8 Mon Sep 17 00:00:00 2001 From: "Ruben Arts(win)" Date: Thu, 16 May 2024 15:17:49 +0200 Subject: [PATCH 25/34] misc: improve ux on target dir feature for windows --- src/project/mod.rs | 107 ++++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/src/project/mod.rs b/src/project/mod.rs index abffda2ff..a14fd2988 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -303,56 +303,44 @@ impl Project { pub fn pixi_dir(&self) -> PathBuf { let default_pixi_dir = self.root.join(consts::PIXI_DIR); - // Custom root directory for target environments if set in configuration. if let Some(custom_root) = self.config().target_environments_directory() { let pixi_dir_name = custom_root.join(format!( "{}-{}", self.name(), xxh3_64(self.root.to_string_lossy().as_bytes()) )); - let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { - tracing::info!( - "Using custom target directory for environments: {}", - pixi_dir_name.display() - ); - // Warn user if environments are found in the default directory - if !default_pixi_dir.is_symlink() { - if default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() { - tracing::warn!( - "Environments found in '{}', this will be ignored and the environment will be installed in the custom target directory '{}'\n\ - \t\tIt's advised to remove the {} folder from the default directory to avoid confusion{}.", - default_pixi_dir.display(), - custom_root.display(), - consts::PIXI_DIR, - if cfg!(windows) { "" } else { " and a symlink to be made. Re-install if needed." } - ); - } else { - if cfg!(windows) { - tracing::warn!("Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory."); - // Write file with warning to the default pixi directory - let warning_file = default_pixi_dir.join("README.txt"); - std::fs::write(&warning_file, format!("Environments are installed in a custom target directory: {}.\nSymlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory.", pixi_dir_name.display())).map_err(|e| println!("Failed to write warning file to {}: {}", warning_file.display(), e)).ok(); - } + let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { - // Create symlink from custom root to default pixi directory - #[cfg(not(windows))] - match symlink(pixi_dir_name.clone(), default_pixi_dir.clone()) { - Ok(_) => tracing::info!("Symlink created successfully from {} to {} folder", custom_root.display(), default_pixi_dir.display()), - Err(e) => println!("Failed to create symlink: {}", e), - } + #[cfg(not(windows))] + if !default_pixi_dir.is_symlink() + && default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() + { + tracing::warn!( + "Environments found in '{}', this will be ignored and the environment will be installed in the custom target directory '{}'\n\ + \t\tIt's advised to remove the {} folder from the default directory to avoid confusion{}.", + default_pixi_dir.display(), + custom_root.display(), + consts::PIXI_DIR, + if cfg!(windows) { "" } else { " and a symlink to be made. Re-install if needed." } + ); + } else { + create_symlink(&pixi_dir_name, &default_pixi_dir); - } } + #[cfg(windows)] + write_warning_file(&default_pixi_dir, &pixi_dir_name); }); return pixi_dir_name; } + tracing::debug!( "Using default root directory: `{}` for target environments.", default_pixi_dir.display() ); + default_pixi_dir } @@ -610,6 +598,63 @@ pub fn find_project_manifest() -> Option { }) } +/// Create a symlink from the default pixi directory to the custom target directory +#[cfg(not(windows))] +fn create_symlink(pixi_dir_name: &PathBuf, default_pixi_dir: &PathBuf) { + symlink(pixi_dir_name.clone(), default_pixi_dir.clone()) + .map_err(|e| { + tracing::error!( + "Failed to create symlink from '{}' to '{}': {}", + pixi_dir_name.display(), + default_pixi_dir.display(), + e + ) + }) + .ok(); +} + +/// Write a warning file to the default pixi directory to inform the user that symlinks are not supported on this platform (Windows). +#[cfg(windows)] +fn write_warning_file(default_pixi_dir: &PathBuf, pixi_dir_name: &Path) { + let warning_file = default_pixi_dir.join("README.txt"); + if warning_file.exists() { + tracing::debug!( + "Symlink warning file already exists at '{}', skipping writing warning file.", + warning_file.display() + ); + return; + } + let warning_message = format!( + "Environments are installed in a custom target directory: {}.\n\ + Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory.", + pixi_dir_name.display() + ); + + // Create directory if it doesn't exist + if let Err(e) = std::fs::create_dir_all(default_pixi_dir) { + tracing::error!( + "Failed to create directory '{}': {}", + default_pixi_dir.display(), + e + ); + return; + } + + // Write warning message to file + match std::fs::write(&warning_file, warning_message.clone()) { + Ok(_) => tracing::info!( + "Symlink warning file written to '{}': {}", + warning_file.display(), + warning_message + ), + Err(e) => tracing::error!( + "Failed to write symlink warning file to '{}': {}", + warning_file.display(), + e + ), + } +} + #[cfg(test)] mod tests { use super::*; From fc12286818d3d570f46c38c9e0cd94b8f1569e71 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 16 May 2024 15:38:04 +0200 Subject: [PATCH 26/34] fix: unix side of the warning system for symlinking --- src/project/mod.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/project/mod.rs b/src/project/mod.rs index a14fd2988..f17003556 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -313,20 +313,17 @@ impl Project { let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { #[cfg(not(windows))] - if !default_pixi_dir.is_symlink() - && default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() - { + if default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() && !default_pixi_dir.is_symlink() { tracing::warn!( - "Environments found in '{}', this will be ignored and the environment will be installed in the custom target directory '{}'\n\ - \t\tIt's advised to remove the {} folder from the default directory to avoid confusion{}.", - default_pixi_dir.display(), - custom_root.display(), - consts::PIXI_DIR, - if cfg!(windows) { "" } else { " and a symlink to be made. Re-install if needed." } - ); + "Environments found in '{}', this will be ignored and the environment will be installed in the custom target directory '{}'\n\ + \t\tIt's advised to remove the {} folder from the default directory to avoid confusion{}.", + default_pixi_dir.display(), + custom_root.display(), + consts::PIXI_DIR, + if cfg!(windows) { "" } else { " and a symlink to be made. Re-install if needed." } + ); } else { create_symlink(&pixi_dir_name, &default_pixi_dir); - } #[cfg(windows)] @@ -600,8 +597,15 @@ pub fn find_project_manifest() -> Option { /// Create a symlink from the default pixi directory to the custom target directory #[cfg(not(windows))] -fn create_symlink(pixi_dir_name: &PathBuf, default_pixi_dir: &PathBuf) { - symlink(pixi_dir_name.clone(), default_pixi_dir.clone()) +fn create_symlink(pixi_dir_name: &Path, default_pixi_dir: &Path) { + if default_pixi_dir.exists() { + tracing::debug!( + "Symlink already exists at '{}', skipping creating symlink.", + default_pixi_dir.display() + ); + return; + } + symlink(pixi_dir_name, default_pixi_dir) .map_err(|e| { tracing::error!( "Failed to create symlink from '{}' to '{}': {}", From 994c9795ba2ca39c719633e6aeff2611eaad26b3 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 22 May 2024 13:46:19 +0200 Subject: [PATCH 27/34] fix: show warning on unknown field --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + src/config.rs | 47 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f506ded2a..937a067c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3329,6 +3329,7 @@ dependencies = [ "self-replace", "serde", "serde-untagged", + "serde_ignored", "serde_json", "serde_with", "serde_yaml", @@ -4600,6 +4601,15 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "serde_ignored" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8e319a36d1b52126a0d608f24e93b2d81297091818cd70625fcf50a15d84ddf" +dependencies = [ + "serde", +] + [[package]] name = "serde_json" version = "1.0.116" diff --git a/Cargo.toml b/Cargo.toml index 06506c7fb..c3c3986ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,7 @@ reqwest-retry = "0.5.0" self-replace = "1.3.7" serde = "1.0.198" serde-untagged = "0.1.5" +serde_ignored = "0.1.10" serde_json = "1.0.116" serde_with = { version = "3.7.0", features = ["indexmap"] } serde_yaml = "0.9.34" diff --git a/src/config.rs b/src/config.rs index 736171396..991f0cb0d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,10 +2,11 @@ use clap::{ArgAction, Parser}; use miette::{miette, Context, IntoDiagnostic}; use rattler_conda_types::{Channel, ChannelConfig, ParseChannelError}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeSet as Set, HashMap}; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::str::FromStr; use url::Url; @@ -133,7 +134,7 @@ pub enum KeyringProvider { } #[derive(Clone, Debug, Deserialize, Serialize, Default)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[serde(rename_all = "kebab-case")] pub struct PyPIConfig { /// The default index URL for PyPI packages. #[serde(default)] @@ -283,14 +284,23 @@ impl Config { /// /// # Returns /// - /// The parsed config + /// The parsed config, and the unused keys /// /// # Errors /// /// Parsing errors #[inline] - pub fn from_toml(toml: &str) -> miette::Result { - toml_edit::de::from_str(toml).into_diagnostic() + pub fn from_toml(toml: &str) -> miette::Result<(Config, Set)> { + let de = toml_edit::de::Deserializer::from_str(toml).into_diagnostic()?; + + // Deserialize the config and collect unused keys + let mut unused_keys = Set::new(); + let config: Config = serde_ignored::deserialize(de, |path| { + unused_keys.insert(path.to_string()); + }) + .into_diagnostic()?; + + Ok((config, unused_keys)) } /// Load the config from the given path. @@ -307,7 +317,24 @@ impl Config { let s = fs::read_to_string(path) .into_diagnostic() .wrap_err(format!("failed to read config from '{}'", path.display()))?; - let mut config = Config::from_toml(&s)?; + + let (mut config, unused_keys) = Config::from_toml(&s)?; + + if !unused_keys.is_empty() { + tracing::warn!( + "Ignoring '{}' in at {}", + console::style( + unused_keys + .iter() + .map(|s| s.as_str()) + .collect::>() + .join(", ") + ) + .yellow(), + path.display() + ); + } + config.loaded_from.push(path.to_path_buf()); tracing::info!("Loaded config from: {}", path.display()); @@ -694,16 +721,18 @@ mod tests { r#"default-channels = ["conda-forge"] tls-no-verify = true target-environments-directory = "{}" +UNUSED = "unused" "#, env!("CARGO_MANIFEST_DIR").replace("\\", "\\\\").as_str() ); - let config = Config::from_toml(toml.as_str()).unwrap(); + let (config, unused) = Config::from_toml(toml.as_str()).unwrap(); assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(true)); assert_eq!( config.target_environments_directory, Some(PathBuf::from(env!("CARGO_MANIFEST_DIR"))) ); + assert!(unused.contains(&"UNUSED".to_string())); } #[test] @@ -742,7 +771,7 @@ target-environments-directory = "{}" extra-index-urls = ["https://pypi.org/simple2"] keyring-provider = "subprocess" "#; - let config = Config::from_toml(toml).unwrap(); + let (config, _) = Config::from_toml(toml).unwrap(); assert_eq!( config.pypi_config().index_url, Some(Url::parse("https://pypi.org/simple").unwrap()) @@ -825,7 +854,7 @@ target-environments-directory = "{}" disable_bzip2 = true disable_zstd = true "#; - let config = Config::from_toml(toml).unwrap(); + let (config, _) = Config::from_toml(toml).unwrap(); assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(false)); assert_eq!( From 9f634c0a65e4bc67c3cc1f25c177d6456ca28aed Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 22 May 2024 13:55:34 +0200 Subject: [PATCH 28/34] docs: fix build --- docs/reference/cli.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 89366de43..e97c3da70 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -351,7 +351,7 @@ List project's packages. Highlighted packages are explicit dependencies. - `--json`: Whether to output in json format. - `--json-pretty`: Whether to output in pretty json format - `--sort-by `: Sorting strategy [default: name] [possible values: size, name, type] -- `--explicit (-x)`: Only list the packages that are explicitly added to the [manifest file](configuration.md). +- `--explicit (-x)`: Only list the packages that are explicitly added to the [manifest file](project_configuration.md). - `--manifest-path `: The path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--environment (-e)`: The environment's packages to list, if non is provided the default environment's packages will be listed. - `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). @@ -695,7 +695,7 @@ Use this command to manage the configuration. - `--global`: Specify management scope to global configuration. - `--local`: Specify management scope to local configuration. -Checkout the [global configuration](../advanced/global_configuration.md) for more information about the locations. +Checkout the [pixi configuration](./pixi_configuration.md) for more information about the locations. ### `config edit` From 038867ab44c8b70005cbd7e4e51c4171358cab70 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 22 May 2024 14:27:35 +0200 Subject: [PATCH 29/34] misc: fix docstrings --- src/config.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/config.rs b/src/config.rs index 991f0cb0d..24262b746 100644 --- a/src/config.rs +++ b/src/config.rs @@ -373,13 +373,9 @@ impl Config { }) } - /// Load the global config file from various global paths. - /// - /// # Returns - /// - /// The loaded global config /// Validate the config file. pub fn validate(&self) -> miette::Result<()> { + // Validate the target environments directory if let Some(target_env_dir) = self.target_environments_directory.clone() { if !target_env_dir.is_absolute() || !target_env_dir.exists() { // The path might exist, but we need it to be absolute because we don't canonicalize it. @@ -390,7 +386,11 @@ impl Config { Ok(()) } - /// Load the global config file from the home directory (~/.pixi/config.toml) + /// Load the global config file from various global paths. + /// + /// # Returns + /// + /// The loaded global config pub fn load_global() -> Config { let mut config = Self::load_system(); From 7b6ec355259a8b7e54da9af79f07128e7b746b3c Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 23 May 2024 13:38:18 +0200 Subject: [PATCH 30/34] wip: renaming to detached-environments --- src/cli/install.rs | 27 +++++--------- src/config.rs | 93 ++++++++++++++++++++++++++++++++++++---------- src/project/mod.rs | 58 ++++++++++++++++------------- 3 files changed, 116 insertions(+), 62 deletions(-) diff --git a/src/cli/install.rs b/src/cli/install.rs index af58f8f4b..06fc65172 100644 --- a/src/cli/install.rs +++ b/src/cli/install.rs @@ -55,28 +55,19 @@ pub async fn execute(args: Args) -> miette::Result<()> { } // Message what's installed - let target_envs_dir_message = if project.config().target_environments_directory.is_some() { - format!( - " in '{}'", - console::style( - project - .config() - .target_environments_directory - .as_ref() - .unwrap() - .display() - ) - .bold() - ) - } else { - "".to_string() - }; + let detached_envs_message = + if let Some(path) = project.config().detached_environments().map(|d| d.path()) { + format!(" in '{}'", console::style(path.display()).bold()) + } else { + "".to_string() + }; + if installed_envs.len() == 1 { eprintln!( "{}The {} environment has been installed{}.", console::style(console::Emoji("✔ ", "")).green(), installed_envs[0].fancy_display(), - target_envs_dir_message + detached_envs_message ); } else { eprintln!( @@ -86,7 +77,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { .iter() .map(|n| n.fancy_display()) .join("\n\t"), - target_envs_dir_message + detached_envs_message ); } diff --git a/src/config.rs b/src/config.rs index 24262b746..62bd2b17a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -150,6 +150,35 @@ pub struct PyPIConfig { pub keyring_provider: Option, } +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +enum DetachedEnvironments { + Boolean(bool), + Path(PathBuf), +} +impl DetachedEnvironments { + fn as_ref(&self) -> &DetachedEnvironments { + self + } + + // Get the path to the detached-environments directory. None means the default directory. + pub(crate) fn path(&self) -> miette::Result> { + match self { + DetachedEnvironments::Path(p) => Ok(Some(p.clone())), + DetachedEnvironments::Boolean(b) if *b => { + let path = get_cache_dir()?.join(consts::ENVIRONMENTS_DIR); + Ok(Some(path)) + } + _ => Ok(None), + } + } +} +impl Default for DetachedEnvironments { + fn default() -> Self { + DetachedEnvironments::Boolean(false) + } +} + impl PyPIConfig { /// Merge the given PyPIConfig into the current one. pub fn merge(self, other: Self) -> Self { @@ -234,10 +263,13 @@ pub struct Config { #[serde(skip_serializing_if = "PyPIConfig::is_default")] pub pypi_config: PyPIConfig, - /// The location of the environments build by pixi + /// The option to specify the directory where detached environments are stored. + /// When using 'true', it defaults to the cache directory. + /// When using a path, it uses the specified path. + /// When using 'false', it disables detached environments, meaning it moves it back to the .pixi folder. #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] - pub target_environments_directory: Option, + pub detached_environments: Option, } impl Default for Config { @@ -252,7 +284,7 @@ impl Default for Config { channel_config: default_channel_config(), repodata_config: None, pypi_config: PyPIConfig::default(), - target_environments_directory: None, + detached_environments: Some(DetachedEnvironments::default()), } } } @@ -375,11 +407,18 @@ impl Config { /// Validate the config file. pub fn validate(&self) -> miette::Result<()> { - // Validate the target environments directory - if let Some(target_env_dir) = self.target_environments_directory.clone() { - if !target_env_dir.is_absolute() || !target_env_dir.exists() { - // The path might exist, but we need it to be absolute because we don't canonicalize it. - return Err(miette!("The `target-environments-directory` path does not exist: {:?}. It needs to be an absolute path to a directory.", target_env_dir)); + // Validate the detached environments directory is set correctly + if let Some(detached_environments) = self.detached_environments.clone() { + match detached_environments { + DetachedEnvironments::Boolean(_) => {} + DetachedEnvironments::Path(path) => { + if !path.is_absolute() { + return Err(miette!( + "The `detached-environments` path must be an absolute path: {}", + path.display() + )); + } + } } } @@ -464,9 +503,7 @@ impl Config { channel_config: other.channel_config, repodata_config: other.repodata_config.or(self.repodata_config), pypi_config: other.pypi_config.merge(self.pypi_config), - target_environments_directory: other - .target_environments_directory - .or(self.target_environments_directory), + detached_environments: other.detached_environments.or(self.detached_environments), } } @@ -530,8 +567,8 @@ impl Config { } /// Retrieve the value for the target_environments_directory field. - pub fn target_environments_directory(&self) -> Option<&Path> { - self.target_environments_directory.as_deref() + pub fn detached_environments(&self) -> Option<&DetachedEnvironments> { + self.detached_environments.as_ref() } /// Modify this config with the given key and value @@ -720,7 +757,7 @@ mod tests { let toml = format!( r#"default-channels = ["conda-forge"] tls-no-verify = true -target-environments-directory = "{}" +detached-environments = "{}" UNUSED = "unused" "#, env!("CARGO_MANIFEST_DIR").replace("\\", "\\\\").as_str() @@ -729,10 +766,26 @@ UNUSED = "unused" assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(true)); assert_eq!( - config.target_environments_directory, + config.detached_environments().unwrap().path().unwrap(), Some(PathBuf::from(env!("CARGO_MANIFEST_DIR"))) ); assert!(unused.contains(&"UNUSED".to_string())); + + let toml = r#"detached-environments = true + "#; + let (config, unused) = Config::from_toml(toml).unwrap(); + assert_eq!( + config + .detached_environments() + .unwrap() + .path() + .unwrap() + .unwrap(), + get_cache_dir() + .unwrap() + .join(consts::ENVIRONMENTS_DIR) + .as_path() + ); } #[test] @@ -790,14 +843,14 @@ UNUSED = "unused" default_channels: vec!["conda-forge".to_string()], channel_config: ChannelConfig::default_with_root_dir(PathBuf::from("/root/dir")), tls_no_verify: Some(true), - target_environments_directory: Some(PathBuf::from("/path/to/envs")), + detached_environments: Some(DetachedEnvironments::Path(PathBuf::from("/path/to/envs"))), ..Default::default() }; config = config.merge_config(other); assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(true)); assert_eq!( - config.target_environments_directory, + config.detached_environments().unwrap().path().unwrap(), Some(PathBuf::from("/path/to/envs")) ); @@ -805,7 +858,9 @@ UNUSED = "unused" default_channels: vec!["channel".to_string()], channel_config: ChannelConfig::default_with_root_dir(PathBuf::from("/root/dir2")), tls_no_verify: Some(false), - target_environments_directory: Some(PathBuf::from("/path/to/envs2")), + detached_environments: Some(DetachedEnvironments::Path(PathBuf::from( + "/path/to/envs2", + ))), ..Default::default() }; @@ -813,7 +868,7 @@ UNUSED = "unused" assert_eq!(config.default_channels, vec!["channel"]); assert_eq!(config.tls_no_verify, Some(false)); assert_eq!( - config.target_environments_directory, + config.detached_environments().unwrap().path().unwrap(), Some(PathBuf::from("/path/to/envs2")) ); diff --git a/src/project/mod.rs b/src/project/mod.rs index e8a9589d1..5dfc51ee1 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -292,10 +292,23 @@ impl Project { /// Returns the pixi directory pub fn pixi_dir(&self) -> PathBuf { - let default_pixi_dir = self.root.join(consts::PIXI_DIR); + self.root.join(consts::PIXI_DIR) + } + + /// Returns the environment directory + pub fn environments_dir(&self) -> PathBuf { + let default_envs_dir = self.pixi_dir().join(consts::ENVIRONMENTS_DIR); + + // Early out if detached-environments is not set + if self.config().detached_environments().is_none() { + return default_envs_dir; + } - if let Some(custom_root) = self.config().target_environments_directory() { - let pixi_dir_name = custom_root.join(format!( + let detached_environments_path = self.config().detached_environments().unwrap().path(); + + // If the detached-environments path is set, use it instead of the default directory. + if let Ok(Some(detached_environments_path)) = detached_environments_path { + let environments_dir_name = detached_environments_path.join(format!( "{}-{}", self.name(), xxh3_64(self.root.to_string_lossy().as_bytes()) @@ -304,37 +317,32 @@ impl Project { let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { #[cfg(not(windows))] - if default_pixi_dir.join(consts::ENVIRONMENTS_DIR).exists() && !default_pixi_dir.is_symlink() { + if default_envs_dir.join(consts::ENVIRONMENTS_DIR).exists() && !default_envs_dir.is_symlink() { tracing::warn!( - "Environments found in '{}', this will be ignored and the environment will be installed in the custom target directory '{}'\n\ + "Environments found in '{}', this will be ignored and the environment will be installed in the detached-environments directory '{}'\n\ \t\tIt's advised to remove the {} folder from the default directory to avoid confusion{}.", - default_pixi_dir.display(), - custom_root.display(), + default_envs_dir.display(), + detached_environments_path.display(), consts::PIXI_DIR, if cfg!(windows) { "" } else { " and a symlink to be made. Re-install if needed." } ); } else { - create_symlink(&pixi_dir_name, &default_pixi_dir); + create_symlink(&environments_dir_name, &default_envs_dir); } #[cfg(windows)] - write_warning_file(&default_pixi_dir, &pixi_dir_name); + write_warning_file(&default_envs_dir, &environments_dir_name); }); - return pixi_dir_name; + return environments_dir_name; } tracing::debug!( - "Using default root directory: `{}` for target environments.", - default_pixi_dir.display() + "Using default root directory: `{}` as environments directory.", + default_envs_dir.display() ); - default_pixi_dir - } - - /// Returns the environment directory - pub fn environments_dir(&self) -> PathBuf { - self.pixi_dir().join(consts::ENVIRONMENTS_DIR) + default_envs_dir } /// Returns the solve group directory @@ -540,8 +548,8 @@ fn create_symlink(pixi_dir_name: &Path, default_pixi_dir: &Path) { /// Write a warning file to the default pixi directory to inform the user that symlinks are not supported on this platform (Windows). #[cfg(windows)] -fn write_warning_file(default_pixi_dir: &PathBuf, pixi_dir_name: &Path) { - let warning_file = default_pixi_dir.join("README.txt"); +fn write_warning_file(default_envs_dir: &PathBuf, envs_dir_name: &Path) { + let warning_file = default_envs_dir.join("README.txt"); if warning_file.exists() { tracing::debug!( "Symlink warning file already exists at '{}', skipping writing warning file.", @@ -550,16 +558,16 @@ fn write_warning_file(default_pixi_dir: &PathBuf, pixi_dir_name: &Path) { return; } let warning_message = format!( - "Environments are installed in a custom target directory: {}.\n\ - Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi') directory.", - pixi_dir_name.display() + "Environments are installed in a custom detached-environments directory: {}.\n\ + Symlinks are not supported on this platform so environments will not be reachable from the default ('.pixi/envs') directory.", + envs_dir_name.display() ); // Create directory if it doesn't exist - if let Err(e) = std::fs::create_dir_all(default_pixi_dir) { + if let Err(e) = std::fs::create_dir_all(default_envs_dir) { tracing::error!( "Failed to create directory '{}': {}", - default_pixi_dir.display(), + default_envs_dir.display(), e ); return; From 3b1d0977292e2b69f8c8616282eb82a1bd3da18d Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 23 May 2024 17:30:27 +0200 Subject: [PATCH 31/34] misc: cleanup and add tests --- src/cli/install.rs | 2 +- src/config.rs | 60 +++++++++++++------ src/project/mod.rs | 15 ++--- .../pixi__config__tests__config_merge.snap | 6 +- 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/src/cli/install.rs b/src/cli/install.rs index 06fc65172..5a15c5974 100644 --- a/src/cli/install.rs +++ b/src/cli/install.rs @@ -56,7 +56,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { // Message what's installed let detached_envs_message = - if let Some(path) = project.config().detached_environments().map(|d| d.path()) { + if let Ok(Some(path)) = project.config().detached_environments().path() { format!(" in '{}'", console::style(path.display()).bold()) } else { "".to_string() diff --git a/src/config.rs b/src/config.rs index 62bd2b17a..23aae4685 100644 --- a/src/config.rs +++ b/src/config.rs @@ -152,17 +152,17 @@ pub struct PyPIConfig { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] #[serde(untagged)] -enum DetachedEnvironments { +pub enum DetachedEnvironments { Boolean(bool), Path(PathBuf), } impl DetachedEnvironments { - fn as_ref(&self) -> &DetachedEnvironments { - self + pub fn is_false(&self) -> bool { + matches!(self, DetachedEnvironments::Boolean(false)) } // Get the path to the detached-environments directory. None means the default directory. - pub(crate) fn path(&self) -> miette::Result> { + pub fn path(&self) -> miette::Result> { match self { DetachedEnvironments::Path(p) => Ok(Some(p.clone())), DetachedEnvironments::Boolean(b) if *b => { @@ -267,7 +267,6 @@ pub struct Config { /// When using 'true', it defaults to the cache directory. /// When using a path, it uses the specified path. /// When using 'false', it disables detached environments, meaning it moves it back to the .pixi folder. - #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub detached_environments: Option, } @@ -298,6 +297,7 @@ impl From for Config { .pypi_keyring_provider .map(|val| PyPIConfig::default().with_keyring(val)) .unwrap_or_default(), + detached_environments: None, ..Default::default() } } @@ -567,8 +567,8 @@ impl Config { } /// Retrieve the value for the target_environments_directory field. - pub fn detached_environments(&self) -> Option<&DetachedEnvironments> { - self.detached_environments.as_ref() + pub fn detached_environments(&self) -> DetachedEnvironments { + self.detached_environments.clone().unwrap_or_default() } /// Modify this config with the given key and value @@ -584,6 +584,7 @@ impl Config { "authentication-override-file", "tls-no-verify", "mirrors", + "detached-environments", "repodata-config", "repodata-config.disable-jlap", "repodata-config.disable-bzip2", @@ -622,6 +623,13 @@ impl Config { .into_diagnostic()? .unwrap_or_default(); } + "detached-environments" => { + self.detached_environments = value.map(|v| match v.as_str() { + "true" => DetachedEnvironments::Boolean(true), + "false" => DetachedEnvironments::Boolean(false), + _ => DetachedEnvironments::Path(PathBuf::from(v)), + }); + } key if key.starts_with("repodata-config") => { if key == "repodata-config" { self.repodata_config = value @@ -766,21 +774,15 @@ UNUSED = "unused" assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(true)); assert_eq!( - config.detached_environments().unwrap().path().unwrap(), + config.detached_environments().path().unwrap(), Some(PathBuf::from(env!("CARGO_MANIFEST_DIR"))) ); assert!(unused.contains(&"UNUSED".to_string())); - let toml = r#"detached-environments = true - "#; - let (config, unused) = Config::from_toml(toml).unwrap(); + let toml = r"detached-environments = true"; + let (config, _) = Config::from_toml(toml).unwrap(); assert_eq!( - config - .detached_environments() - .unwrap() - .path() - .unwrap() - .unwrap(), + config.detached_environments().path().unwrap().unwrap(), get_cache_dir() .unwrap() .join(consts::ENVIRONMENTS_DIR) @@ -850,7 +852,7 @@ UNUSED = "unused" assert_eq!(config.default_channels, vec!["conda-forge"]); assert_eq!(config.tls_no_verify, Some(true)); assert_eq!( - config.detached_environments().unwrap().path().unwrap(), + config.detached_environments().path().unwrap(), Some(PathBuf::from("/path/to/envs")) ); @@ -868,7 +870,7 @@ UNUSED = "unused" assert_eq!(config.default_channels, vec!["channel"]); assert_eq!(config.tls_no_verify, Some(false)); assert_eq!( - config.detached_environments().unwrap().path().unwrap(), + config.detached_environments().path().unwrap(), Some(PathBuf::from("/path/to/envs2")) ); @@ -880,6 +882,7 @@ UNUSED = "unused" let config_2 = Config::from_path(&d.join("config_2.toml")).unwrap(); let config_2 = Config { channel_config: ChannelConfig::default_with_root_dir(PathBuf::from("/root/dir")), + detached_environments: Some(DetachedEnvironments::Boolean(true)), ..config_2 }; @@ -969,6 +972,25 @@ UNUSED = "unused" Some(PathBuf::from("/path/to/your/override.json")) ); + config + .set("detached-environments", Some("true".to_string())) + .unwrap(); + assert_eq!( + config.detached_environments().path().unwrap().unwrap(), + get_cache_dir() + .unwrap() + .join(consts::ENVIRONMENTS_DIR) + .as_path() + ); + + config + .set("detached-environments", Some("/path/to/envs".to_string())) + .unwrap(); + assert_eq!( + config.detached_environments().path().unwrap(), + Some(PathBuf::from("/path/to/envs")) + ); + config .set("mirrors", Some(r#"{"https://conda.anaconda.org/conda-forge": ["https://prefix.dev/conda-forge"]}"#.to_string())) .unwrap(); diff --git a/src/project/mod.rs b/src/project/mod.rs index 5dfc51ee1..2aa4602b1 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -300,14 +300,12 @@ impl Project { let default_envs_dir = self.pixi_dir().join(consts::ENVIRONMENTS_DIR); // Early out if detached-environments is not set - if self.config().detached_environments().is_none() { + if self.config().detached_environments().is_false() { return default_envs_dir; } - let detached_environments_path = self.config().detached_environments().unwrap().path(); - // If the detached-environments path is set, use it instead of the default directory. - if let Ok(Some(detached_environments_path)) = detached_environments_path { + if let Ok(Some(detached_environments_path)) = self.config().detached_environments().path() { let environments_dir_name = detached_environments_path.join(format!( "{}-{}", self.name(), @@ -317,14 +315,13 @@ impl Project { let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { #[cfg(not(windows))] - if default_envs_dir.join(consts::ENVIRONMENTS_DIR).exists() && !default_envs_dir.is_symlink() { + if default_envs_dir.exists() && !default_envs_dir.is_symlink() { tracing::warn!( - "Environments found in '{}', this will be ignored and the environment will be installed in the detached-environments directory '{}'\n\ - \t\tIt's advised to remove the {} folder from the default directory to avoid confusion{}.", + "Environments found in '{}', this will be ignored and the environment will be installed in the 'detached-environments' directory: '{}'. It's advised to remove the {} folder from the default directory to avoid confusion{}.", default_envs_dir.display(), detached_environments_path.display(), - consts::PIXI_DIR, - if cfg!(windows) { "" } else { " and a symlink to be made. Re-install if needed." } + format!("{}/{}", consts::PIXI_DIR, consts::ENVIRONMENTS_DIR), + if cfg!(windows) { "" } else { " as a symlink can be made, please re-install after removal." } ); } else { create_symlink(&environments_dir_name, &default_envs_dir); diff --git a/src/snapshots/pixi__config__tests__config_merge.snap b/src/snapshots/pixi__config__tests__config_merge.snap index 86043da8f..a5957b955 100644 --- a/src/snapshots/pixi__config__tests__config_merge.snap +++ b/src/snapshots/pixi__config__tests__config_merge.snap @@ -54,5 +54,9 @@ Config { extra_index_urls: [], keyring_provider: None, }, - target_environments_directory: None, + detached_environments: Some( + Boolean( + true, + ), + ), } From d13e9782dcdb48b54797ea134385db84c463d8f1 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Fri, 24 May 2024 07:54:02 +0200 Subject: [PATCH 32/34] fix: tests --- tests/install_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index 944ad2c6c..14fc20138 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -10,7 +10,7 @@ use crate::common::package_database::{Package, PackageDatabase}; use common::{LockFileExt, PixiControl}; use pixi::cli::run::Args; use pixi::cli::{run, LockFileUsageArgs}; -use pixi::config::Config; +use pixi::config::{Config, DetachedEnvironments}; use pixi::consts::{DEFAULT_ENVIRONMENT_NAME, PIXI_UV_INSTALLER}; use pixi::FeatureName; use rattler_conda_types::Platform; @@ -132,7 +132,7 @@ async fn install_locked_with_config() { // Overwrite install location to a target directory let mut config = Config::default(); let target_dir = pixi.project_path().join("target"); - config.target_environments_directory = Some(target_dir.clone()); + config.detached_environments = Some(DetachedEnvironments::Path(target_dir.clone())); create_dir_all(target_dir.clone()).unwrap(); let config_path = pixi.project().unwrap().pixi_dir().join("config.toml"); From ebeec73033bde1257d935731205552673b66cb65 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Fri, 24 May 2024 09:20:04 +0200 Subject: [PATCH 33/34] fix: also move solve groups --- src/project/mod.rs | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/project/mod.rs b/src/project/mod.rs index 2aa4602b1..2f4f11c56 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -295,6 +295,19 @@ impl Project { self.root.join(consts::PIXI_DIR) } + /// Create the detached-environments path for this project if it is set in the config + fn detached_environments_path(&self) -> Option { + if let Ok(Some(detached_environments_path)) = self.config().detached_environments().path() { + Some(detached_environments_path.join(format!( + "{}-{}", + self.name(), + xxh3_64(self.root.to_string_lossy().as_bytes()) + ))) + } else { + None + } + } + /// Returns the environment directory pub fn environments_dir(&self) -> PathBuf { let default_envs_dir = self.pixi_dir().join(consts::ENVIRONMENTS_DIR); @@ -305,13 +318,9 @@ impl Project { } // If the detached-environments path is set, use it instead of the default directory. - if let Ok(Some(detached_environments_path)) = self.config().detached_environments().path() { - let environments_dir_name = detached_environments_path.join(format!( - "{}-{}", - self.name(), - xxh3_64(self.root.to_string_lossy().as_bytes()) - )); - + if let Some(detached_environments_path) = self.detached_environments_path() { + let detached_environments_path = + detached_environments_path.join(consts::ENVIRONMENTS_DIR); let _ = CUSTOM_TARGET_DIR_WARN.get_or_init(|| { #[cfg(not(windows))] @@ -319,19 +328,19 @@ impl Project { tracing::warn!( "Environments found in '{}', this will be ignored and the environment will be installed in the 'detached-environments' directory: '{}'. It's advised to remove the {} folder from the default directory to avoid confusion{}.", default_envs_dir.display(), - detached_environments_path.display(), + detached_environments_path.parent().expect("path should have parent").display(), format!("{}/{}", consts::PIXI_DIR, consts::ENVIRONMENTS_DIR), if cfg!(windows) { "" } else { " as a symlink can be made, please re-install after removal." } ); } else { - create_symlink(&environments_dir_name, &default_envs_dir); + create_symlink(&detached_environments_path, &default_envs_dir); } #[cfg(windows)] - write_warning_file(&default_envs_dir, &environments_dir_name); + write_warning_file(&default_envs_dir, &detached_environments_path); }); - return environments_dir_name; + return detached_environments_path; } tracing::debug!( @@ -342,8 +351,12 @@ impl Project { default_envs_dir } - /// Returns the solve group directory + /// Returns the solve group environments directory pub fn solve_group_environments_dir(&self) -> PathBuf { + // If the detached-environments path is set, use it instead of the default directory. + if let Some(detached_environments_path) = self.detached_environments_path() { + return detached_environments_path.join(consts::SOLVE_GROUP_ENVIRONMENTS_DIR); + } self.pixi_dir().join(consts::SOLVE_GROUP_ENVIRONMENTS_DIR) } From c53b713263c5f8ab767b3a93c7a7ae21f0fbd6a5 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Fri, 24 May 2024 09:20:23 +0200 Subject: [PATCH 34/34] docs: update documentation --- docs/reference/cli.md | 2 ++ docs/reference/pixi_configuration.md | 33 +++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index e97c3da70..909f9a0e6 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -758,6 +758,8 @@ Set a configuration key to a value. pixi config set default-channels '["conda-forge", "bioconda"]' pixi config set --global mirrors '{"https://conda.anaconda.org/": ["https://prefix.dev/conda-forge"]}' pixi config set repodata-config.disable-zstd true --system +pixi config set --global detached-environments "/opt/pixi/envs" +pixi config set detached-environments false ``` ### `config unset` diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index 75c7c855b..0f2c3053c 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -103,21 +103,48 @@ Read more in the authentication section. authentication-override-file = "/path/to/your/override.json" ``` -### `target-environment-directory` -The directory where pixi stores the project environments, what would normally be placed in the `.pixi` folder in a project's root. +### `detached-environments` +The directory where pixi stores the project environments, what would normally be placed in the `.pixi/envs` folder in a project's root. It doesn't affect the environments built for `pixi global`. The location of environments created for a `pixi global` installation can be controlled using the `PIXI_HOME` environment variable. !!! warning We recommend against using this because any environment created for a project is no longer placed in the same folder as the project. This creates a disconnect between the project and its environments and manual cleanup of the environments is required when deleting the project. + However, in some cases, this option can still be very useful, for instance to: - force the installation on a specific filesystem/drive. - install environments locally but keep the project on a network drive. - let a system-administrator have more control over all environments on a system. +This field can consist of two types of input. + +- A boolean value, `true` or `false`, which will enable or disable the feature respectively. (not `"true"` or `"false"`, this is read as `false`) +- A string value, which will be the absolute path to the directory where the environments will be stored. + ```toml title="config.toml" -target-environment-diretory = "/opt/pixi/envs" +detached-environments = true +``` +or: +```toml title="config.toml" +detached-environments = "/opt/pixi/envs" +``` + +The environments will be stored in the [cache directory](../features/environment.md#caching) when this option is `true`. +When you specify a custom path the environments will be stored in that directory. + +The resulting directory structure will look like this: +```toml title="config.toml" +detached-environments = "/opt/pixi/envs" +``` +```shell +/opt/pixi/envs +├── pixi-6837172896226367631 +│ └── envs +└── NAME_OF_PROJECT-HASH_OF_ORIGINAL_PATH + ├── envs # the runnable environments + └── solve-group-envs # If there are solve groups + ``` ### `mirrors`