From abcc0deb564f3d3b76e37280c19d015a64eb555e Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Wed, 22 May 2024 07:52:12 -0400 Subject: [PATCH 1/6] Export conda environment.yml Adds an export command group, and a subcommand for exporting in Conda environment.yml files. Currently it just exports the direct dependency names, but I could see adding flags to include all locked dependencies. It also currently does not export the versions, but that could also be a flag to select if it should export the specs as in the manifest, or as locked. works on #800 --- src/cli/export/conda.rs | 80 +++++++++++++++++++++++++++++ src/cli/export/mod.rs | 23 +++++++++ src/cli/mod.rs | 4 ++ src/utils/conda_environment_file.rs | 12 ++--- 4 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 src/cli/export/conda.rs create mode 100644 src/cli/export/mod.rs diff --git a/src/cli/export/conda.rs b/src/cli/export/conda.rs new file mode 100644 index 000000000..a2f712fa0 --- /dev/null +++ b/src/cli/export/conda.rs @@ -0,0 +1,80 @@ +use std::path::PathBuf; + +use clap::Parser; + +use itertools::Itertools; +use miette::IntoDiagnostic; +use rattler_conda_types::Platform; + +use crate::utils::conda_environment_file::{CondaEnvDep, CondaEnvFile}; +use crate::{HasFeatures, Project}; + +/// Exports a projects dependencies as an environment.yml +/// +/// The environment is printed to standard out +#[derive(Debug, Parser)] +#[clap(arg_required_else_help = false)] +pub struct Args { + /// The platform to list packages for. Defaults to the current platform. + #[arg(long)] + pub platform: Option, + + /// The path to 'pixi.toml' or 'pyproject.toml' + #[arg(long)] + pub manifest_path: Option, + + /// The environment to list packages for. Defaults to the default environment. + #[arg(short, long)] + pub environment: Option, + + /// Name for environment + #[arg(short, long)] + pub name: Option, +} + +pub async fn execute(args: Args) -> miette::Result<()> { + let project = Project::load_or_else_discover(args.manifest_path.as_deref())?; + let environment = project.environment_from_name_or_env_var(args.environment)?; + + let platform = args.platform.unwrap_or_else(|| environment.best_platform()); + + let name = match args.name { + Some(arg_name) => arg_name, + None => format!("{}-{}-{}", project.name(), environment.name(), platform), + }; + + let channels = environment + .channels() + .into_iter() + .map(|channel| channel.name().to_string()) + .collect_vec(); + + let mut dependencies = environment + .dependencies(None, Some(platform)) + .into_specs() + .map(|(name, _spec)| CondaEnvDep::Conda(name.as_source().to_string())) + .collect_vec(); + + let pypi_dependencies = environment + .pypi_dependencies(Some(platform)) + .into_specs() + .map(|(name, _spec)| name.as_source().to_string()) + .collect_vec(); + + if !pypi_dependencies.is_empty() { + dependencies.push(CondaEnvDep::Pip { + pip: pypi_dependencies, + }); + } + + let env_file = CondaEnvFile { + name: Some(name), + channels, + dependencies, + }; + + let env_string = serde_yaml::to_string(&env_file).into_diagnostic()?; + println!("{}", env_string); + + Ok(()) +} diff --git a/src/cli/export/mod.rs b/src/cli/export/mod.rs new file mode 100644 index 000000000..ee2133155 --- /dev/null +++ b/src/cli/export/mod.rs @@ -0,0 +1,23 @@ +use clap::Parser; + +mod conda; + +#[derive(Debug, Parser)] +pub enum Command { + #[clap(alias = "c")] + Conda(conda::Args), +} + +/// Subcommand for exporting dependencies to additional formats +#[derive(Debug, Parser)] +pub struct Args { + #[command(subcommand)] + command: Command, +} + +pub async fn execute(cmd: Args) -> miette::Result<()> { + match cmd.command { + Command::Conda(args) => conda::execute(args).await?, + }; + Ok(()) +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index c49048101..6b3b35cc4 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -12,6 +12,7 @@ use tracing_subscriber::{filter::LevelFilter, util::SubscriberInitExt, EnvFilter pub mod add; pub mod completion; pub mod config; +pub mod export; pub mod global; pub mod has_specs; pub mod info; @@ -103,6 +104,8 @@ pub enum Command { List(list::Args), #[clap(visible_alias = "t")] Tree(tree::Args), + #[clap(visible_alias = "e")] + Export(export::Args), // Global level commands #[clap(visible_alias = "g")] @@ -276,6 +279,7 @@ pub async fn execute_command(command: Command) -> miette::Result<()> { Command::SelfUpdate(cmd) => self_update::execute(cmd).await, Command::List(cmd) => list::execute(cmd).await, Command::Tree(cmd) => tree::execute(cmd).await, + Command::Export(cmd) => export::execute(cmd).await, } } diff --git a/src/utils/conda_environment_file.rs b/src/utils/conda_environment_file.rs index 94a5d1181..6aa012af5 100644 --- a/src/utils/conda_environment_file.rs +++ b/src/utils/conda_environment_file.rs @@ -2,22 +2,22 @@ use itertools::Itertools; use miette::IntoDiagnostic; use rattler_conda_types::ParseStrictness::Lenient; use rattler_conda_types::{Channel, MatchSpec}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::str::FromStr; use std::{io::BufRead, path::Path, sync::Arc}; use crate::config::Config; -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct CondaEnvFile { #[serde(default)] - name: Option, + pub name: Option, #[serde(default)] - channels: Vec, - dependencies: Vec, + pub channels: Vec, + pub dependencies: Vec, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] pub enum CondaEnvDep { Conda(String), From ffb776b1892e7176cd506629b79b4c5a9c266aa8 Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Wed, 22 May 2024 09:07:53 -0400 Subject: [PATCH 2/6] Move export commands to project command group --- src/cli/mod.rs | 4 ---- src/cli/{ => project}/export/conda.rs | 0 src/cli/{ => project}/export/mod.rs | 2 +- src/cli/project/mod.rs | 3 +++ 4 files changed, 4 insertions(+), 5 deletions(-) rename src/cli/{ => project}/export/conda.rs (100%) rename src/cli/{ => project}/export/mod.rs (85%) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 6b3b35cc4..c49048101 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -12,7 +12,6 @@ use tracing_subscriber::{filter::LevelFilter, util::SubscriberInitExt, EnvFilter pub mod add; pub mod completion; pub mod config; -pub mod export; pub mod global; pub mod has_specs; pub mod info; @@ -104,8 +103,6 @@ pub enum Command { List(list::Args), #[clap(visible_alias = "t")] Tree(tree::Args), - #[clap(visible_alias = "e")] - Export(export::Args), // Global level commands #[clap(visible_alias = "g")] @@ -279,7 +276,6 @@ pub async fn execute_command(command: Command) -> miette::Result<()> { Command::SelfUpdate(cmd) => self_update::execute(cmd).await, Command::List(cmd) => list::execute(cmd).await, Command::Tree(cmd) => tree::execute(cmd).await, - Command::Export(cmd) => export::execute(cmd).await, } } diff --git a/src/cli/export/conda.rs b/src/cli/project/export/conda.rs similarity index 100% rename from src/cli/export/conda.rs rename to src/cli/project/export/conda.rs diff --git a/src/cli/export/mod.rs b/src/cli/project/export/mod.rs similarity index 85% rename from src/cli/export/mod.rs rename to src/cli/project/export/mod.rs index ee2133155..024250b1d 100644 --- a/src/cli/export/mod.rs +++ b/src/cli/project/export/mod.rs @@ -8,7 +8,7 @@ pub enum Command { Conda(conda::Args), } -/// Subcommand for exporting dependencies to additional formats +/// Commands for exporting dependencies to additional formats #[derive(Debug, Parser)] pub struct Args { #[command(subcommand)] diff --git a/src/cli/project/mod.rs b/src/cli/project/mod.rs index 3d1bbf4db..308b2b273 100644 --- a/src/cli/project/mod.rs +++ b/src/cli/project/mod.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; pub mod channel; pub mod description; +pub mod export; pub mod platform; pub mod version; @@ -10,6 +11,7 @@ pub mod version; pub enum Command { Channel(channel::Args), Description(description::Args), + Export(export::Args), Platform(platform::Args), Version(version::Args), } @@ -28,6 +30,7 @@ pub async fn execute(cmd: Args) -> miette::Result<()> { match cmd.command { Command::Channel(args) => channel::execute(args).await?, Command::Description(args) => description::execute(args).await?, + Command::Export(cmd) => export::execute(cmd).await?, Command::Platform(args) => platform::execute(args).await?, Command::Version(args) => version::execute(args).await?, }; From 6243e53dcdbd01c4b6457e374d58cf0a2a6163a3 Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Wed, 22 May 2024 10:56:38 -0400 Subject: [PATCH 3/6] Format conda dependencies with matchspecs from manifest --- src/cli/project/export/conda.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/cli/project/export/conda.rs b/src/cli/project/export/conda.rs index a2f712fa0..c5449035d 100644 --- a/src/cli/project/export/conda.rs +++ b/src/cli/project/export/conda.rs @@ -4,11 +4,19 @@ use clap::Parser; use itertools::Itertools; use miette::IntoDiagnostic; -use rattler_conda_types::Platform; +use rattler_conda_types::{MatchSpec, Platform}; use crate::utils::conda_environment_file::{CondaEnvDep, CondaEnvFile}; use crate::{HasFeatures, Project}; +// enum to select version spec formatting +#[derive(clap::ValueEnum, Clone, Debug)] +pub enum VersionSpec { + Manifest, + // Locked, + None, +} + /// Exports a projects dependencies as an environment.yml /// /// The environment is printed to standard out @@ -30,6 +38,10 @@ pub struct Args { /// Name for environment #[arg(short, long)] pub name: Option, + + /// Dependency spec output method + #[arg(long, default_value = "manifest", value_enum)] + pub version_spec: VersionSpec, } pub async fn execute(args: Args) -> miette::Result<()> { @@ -52,7 +64,12 @@ pub async fn execute(args: Args) -> miette::Result<()> { let mut dependencies = environment .dependencies(None, Some(platform)) .into_specs() - .map(|(name, _spec)| CondaEnvDep::Conda(name.as_source().to_string())) + .map(|(name, spec)| match args.version_spec { + VersionSpec::Manifest => { + CondaEnvDep::Conda(MatchSpec::from_nameless(spec, Some(name)).to_string()) + } + _ => CondaEnvDep::Conda(name.as_source().to_string()), + }) .collect_vec(); let pypi_dependencies = environment From 4b10624a7c1cbb7ee6fdb301ff270b2536ff3f7a Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Wed, 22 May 2024 12:19:46 -0400 Subject: [PATCH 4/6] Initial pass at PyPI dependency specs --- src/cli/project/export/conda.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cli/project/export/conda.rs b/src/cli/project/export/conda.rs index c5449035d..7bcc6e515 100644 --- a/src/cli/project/export/conda.rs +++ b/src/cli/project/export/conda.rs @@ -75,7 +75,16 @@ pub async fn execute(args: Args) -> miette::Result<()> { let pypi_dependencies = environment .pypi_dependencies(Some(platform)) .into_specs() - .map(|(name, _spec)| name.as_source().to_string()) + .map(|(name, spec)| match args.version_spec { + VersionSpec::Manifest => { + let requirement = spec + .as_pep508(name.as_normalized(), project.root()) + .into_diagnostic() + .unwrap(); + requirement.to_string() + } + _ => name.as_source().to_string(), + }) .collect_vec(); if !pypi_dependencies.is_empty() { From 763bcd6ae7e2c3758f4cbcb86098caada372a9ae Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Thu, 23 May 2024 08:26:55 -0400 Subject: [PATCH 5/6] Match editable pypi packages and format those specifically --- src/cli/project/export/conda.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/cli/project/export/conda.rs b/src/cli/project/export/conda.rs index 7bcc6e515..4b38b8dcd 100644 --- a/src/cli/project/export/conda.rs +++ b/src/cli/project/export/conda.rs @@ -7,7 +7,7 @@ use miette::IntoDiagnostic; use rattler_conda_types::{MatchSpec, Platform}; use crate::utils::conda_environment_file::{CondaEnvDep, CondaEnvFile}; -use crate::{HasFeatures, Project}; +use crate::{project, HasFeatures, Project}; // enum to select version spec formatting #[derive(clap::ValueEnum, Clone, Debug)] @@ -81,7 +81,19 @@ pub async fn execute(args: Args) -> miette::Result<()> { .as_pep508(name.as_normalized(), project.root()) .into_diagnostic() .unwrap(); - requirement.to_string() + return match &requirement { + project::manifest::python::RequirementOrEditable::Editable( + _package_name, + requirements_txt, + ) => { + let relative_path = requirements_txt + .path + .as_path() + .strip_prefix(project.manifest_path().parent().unwrap()); + format!("-e ./{}", relative_path.unwrap().to_string_lossy()) + } + _ => requirement.to_string(), + }; } _ => name.as_source().to_string(), }) From 47a9c69cc7c455cf26e2b84de42379a008d0db10 Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Thu, 23 May 2024 13:44:51 -0400 Subject: [PATCH 6/6] Add the ability to export the locked environments --- src/cli/project/export/conda.rs | 82 ++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/src/cli/project/export/conda.rs b/src/cli/project/export/conda.rs index 4b38b8dcd..f24048007 100644 --- a/src/cli/project/export/conda.rs +++ b/src/cli/project/export/conda.rs @@ -6,6 +6,8 @@ use itertools::Itertools; use miette::IntoDiagnostic; use rattler_conda_types::{MatchSpec, Platform}; +use crate::cli::LockFileUsageArgs; +use crate::lock_file::UpdateLockFileOptions; use crate::utils::conda_environment_file::{CondaEnvDep, CondaEnvFile}; use crate::{project, HasFeatures, Project}; @@ -13,7 +15,7 @@ use crate::{project, HasFeatures, Project}; #[derive(clap::ValueEnum, Clone, Debug)] pub enum VersionSpec { Manifest, - // Locked, + Locked, None, } @@ -42,6 +44,13 @@ pub struct Args { /// Dependency spec output method #[arg(long, default_value = "manifest", value_enum)] pub version_spec: VersionSpec, + + #[clap(flatten)] + pub lock_file_usage: LockFileUsageArgs, + + /// Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. + #[arg(long)] + pub no_install: bool, } pub async fn execute(args: Args) -> miette::Result<()> { @@ -61,6 +70,77 @@ pub async fn execute(args: Args) -> miette::Result<()> { .map(|channel| channel.name().to_string()) .collect_vec(); + if let VersionSpec::Locked = args.version_spec { + let lock_file = project + .up_to_date_lock_file(UpdateLockFileOptions { + lock_file_usage: args.lock_file_usage.into(), + no_install: args.no_install, + ..UpdateLockFileOptions::default() + }) + .await?; + + let locked_deps = lock_file + .lock_file + .environment(environment.name().as_str()) + .and_then(|env| env.packages(platform).map(Vec::from_iter)) + .unwrap_or_default(); + + let mut dependencies = locked_deps + .iter() + .filter_map(|d| d.as_conda()) + .map(|d| CondaEnvDep::Conda(d.package_record().to_string())) + .collect_vec(); + + let mut pypi_dependencies = locked_deps + .iter() + .filter_map(|d| d.as_pypi()) + .filter(|d| !d.is_editable()) + .map(|d| format!("{}={}", d.data().package.name, d.data().package.version)) + .collect_vec(); + + let editable_dependencies = environment + .pypi_dependencies(Some(platform)) + .into_specs() + .filter_map(|(name, spec)| { + let requirement = spec + .as_pep508(name.as_normalized(), project.root()) + .into_diagnostic() + .unwrap(); + if let project::manifest::python::RequirementOrEditable::Editable( + _package_name, + requirements_txt, + ) = &requirement + { + let relative_path = requirements_txt + .path + .as_path() + .strip_prefix(project.manifest_path().parent().unwrap()); + return Some(format!("-e ./{}", relative_path.unwrap().to_string_lossy())); + } + None + }) + .collect_vec(); + + pypi_dependencies.extend(editable_dependencies); + + if !pypi_dependencies.is_empty() { + dependencies.push(CondaEnvDep::Pip { + pip: pypi_dependencies, + }); + } + + let env_file = CondaEnvFile { + name: Some(name), + channels, + dependencies, + }; + + let env_string = serde_yaml::to_string(&env_file).into_diagnostic()?; + println!("{}", env_string); + + return Ok(()); + } + let mut dependencies = environment .dependencies(None, Some(platform)) .into_specs()