From d7d43582010a46c0f64cc0c8131d0e9d8bc1b96e Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 18 Jul 2024 14:56:44 +1000 Subject: [PATCH 1/4] Update refresh request --- Cargo.lock | 1 + crates/pet-conda/src/lib.rs | 6 +- crates/pet-core/Cargo.toml | 1 + crates/pet-core/src/lib.rs | 21 +++- crates/pet-core/src/python_environment.rs | 6 +- crates/pet-homebrew/src/lib.rs | 6 +- crates/pet-linux-global-python/src/lib.rs | 6 +- crates/pet-mac-commandlinetools/src/lib.rs | 20 +++- crates/pet-mac-python-org/src/lib.rs | 6 +- crates/pet-mac-xcode/src/lib.rs | 19 +++- crates/pet-pipenv/src/lib.rs | 5 +- crates/pet-poetry/src/lib.rs | 6 +- crates/pet-pyenv/src/lib.rs | 7 +- crates/pet-python-utils/src/env.rs | 5 +- crates/pet-python-utils/src/fs_cache.rs | 2 +- crates/pet-reporter/src/jsonrpc.rs | 23 +++- crates/pet-reporter/src/stdio.rs | 7 +- crates/pet-venv/src/lib.rs | 6 +- crates/pet-virtualenv/src/lib.rs | 6 +- crates/pet-virtualenvwrapper/src/lib.rs | 6 +- crates/pet-windows-registry/src/lib.rs | 11 +- crates/pet-windows-store/src/lib.rs | 5 +- crates/pet/src/find.rs | 88 ++++++++++----- crates/pet/src/jsonrpc.rs | 57 ++++++++-- crates/pet/src/lib.rs | 122 ++++++++------------- crates/pet/src/main.rs | 64 ++++++----- docs/JSONRPC.md | 101 +++++------------ 27 files changed, 349 insertions(+), 264 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fde7bebd..04e26ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,6 +391,7 @@ dependencies = [ name = "pet-core" version = "0.1.0" dependencies = [ + "clap", "lazy_static", "log", "msvc_spectre_libs", diff --git a/crates/pet-conda/src/lib.rs b/crates/pet-conda/src/lib.rs index 00ac78f6..6e035a1a 100644 --- a/crates/pet-conda/src/lib.rs +++ b/crates/pet-conda/src/lib.rs @@ -15,7 +15,7 @@ use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_fs::path::norm_case; use serde::{Deserialize, Serialize}; @@ -203,8 +203,8 @@ impl Conda { } impl Locator for Conda { - fn get_name(&self) -> &'static str { - "Conda" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::Conda } fn configure(&self, config: &pet_core::Configuration) { if let Some(ref conda_exe) = config.conda_executable { diff --git a/crates/pet-core/Cargo.toml b/crates/pet-core/Cargo.toml index 0e14864d..491cc758 100644 --- a/crates/pet-core/Cargo.toml +++ b/crates/pet-core/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" msvc_spectre_libs = { version = "0.1.1", features = ["error"] } [dependencies] +clap = { version = "4.5.4", features = ["derive", "cargo"] } pet-fs = { path = "../pet-fs" } serde = { version = "1.0.152", features = ["derive"] } lazy_static = "1.4.0" diff --git a/crates/pet-core/src/lib.rs b/crates/pet-core/src/lib.rs index 81606320..033ec44b 100644 --- a/crates/pet-core/src/lib.rs +++ b/crates/pet-core/src/lib.rs @@ -27,6 +27,7 @@ pub struct LocatorResult { pub struct Configuration { /// These are paths like workspace folders, where we can look for environments. pub workspace_directories: Option>, + pub executables: Option>, pub conda_executable: Option, pub poetry_executable: Option, /// Custom locations where environments can be found. @@ -37,9 +38,27 @@ pub struct Configuration { pub cache_directory: Option, } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum LocatorKind { + Conda, + Homebrew, + LinuxGlobal, + MacCommandLineTools, + MacPythonOrg, + MacXCode, + PipEnv, + Poetry, + PyEnv, + Venv, + VirtualEnv, + VirtualEnvWrapper, + WindowsRegistry, + WindowsStore, +} + pub trait Locator: Send + Sync { /// Returns the name of the locator. - fn get_name(&self) -> &'static str; + fn get_kind(&self) -> LocatorKind; /// Configures the locator with the given configuration. /// Override this method if you need to have some custom configuration. /// E.g. storing some of the configuration information in the locator. diff --git a/crates/pet-core/src/python_environment.rs b/crates/pet-core/src/python_environment.rs index eb144581..7cb18a44 100644 --- a/crates/pet-core/src/python_environment.rs +++ b/crates/pet-core/src/python_environment.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use clap::{Parser, ValueEnum}; use log::error; use pet_fs::path::norm_case; use serde::{Deserialize, Serialize}; @@ -8,11 +9,11 @@ use std::path::PathBuf; use crate::{arch::Architecture, manager::EnvManager}; -#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug, Hash)] +#[derive(Parser, ValueEnum, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum PythonEnvironmentKind { Conda, Homebrew, - Pyenv, // Relates to Python installations in pyenv that are from Python org. + Pyenv, GlobalPaths, // Python found in global locations like PATH, /usr/bin etc. PyenvVirtualEnv, // Pyenv virtualenvs. Pipenv, @@ -21,7 +22,6 @@ pub enum PythonEnvironmentKind { MacCommandLineTools, LinuxGlobal, MacXCode, - Unknown, Venv, VirtualEnv, VirtualEnvWrapper, diff --git a/crates/pet-homebrew/src/lib.rs b/crates/pet-homebrew/src/lib.rs index 1aedc0bf..54c65b7a 100644 --- a/crates/pet-homebrew/src/lib.rs +++ b/crates/pet-homebrew/src/lib.rs @@ -9,7 +9,7 @@ use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_fs::path::resolve_symlink; use pet_python_utils::executable::find_executables; @@ -110,8 +110,8 @@ fn from(env: &PythonEnv) -> Option { } impl Locator for Homebrew { - fn get_name(&self) -> &'static str { - "Homebrew" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::Homebrew } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::Homebrew] diff --git a/crates/pet-linux-global-python/src/lib.rs b/crates/pet-linux-global-python/src/lib.rs index e62f1bf1..2fc76e94 100644 --- a/crates/pet-linux-global-python/src/lib.rs +++ b/crates/pet-linux-global-python/src/lib.rs @@ -14,7 +14,7 @@ use pet_core::{ env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_fs::path::resolve_symlink; use pet_python_utils::{env::ResolvedPythonEnv, executable::find_executables}; @@ -53,8 +53,8 @@ impl Default for LinuxGlobalPython { } } impl Locator for LinuxGlobalPython { - fn get_name(&self) -> &'static str { - "LinuxGlobalPython" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::LinuxGlobal } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::LinuxGlobal] diff --git a/crates/pet-mac-commandlinetools/src/lib.rs b/crates/pet-mac-commandlinetools/src/lib.rs index 382dc6c1..ebcdf9e8 100644 --- a/crates/pet-mac-commandlinetools/src/lib.rs +++ b/crates/pet-mac-commandlinetools/src/lib.rs @@ -2,10 +2,11 @@ // Licensed under the MIT License. use pet_core::{ + arch::Architecture, env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_fs::path::resolve_symlink; use pet_python_utils::version; @@ -26,8 +27,8 @@ impl Default for MacCmdLineTools { } } impl Locator for MacCmdLineTools { - fn get_name(&self) -> &'static str { - "MacCmdLineTools" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::MacCommandLineTools } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::MacCommandLineTools] @@ -57,6 +58,7 @@ impl Locator for MacCmdLineTools { let mut version = env.version.clone(); let mut prefix = env.prefix.clone(); let mut symlinks = vec![env.executable.clone()]; + let mut arch = None; let existing_symlinks = env.symlinks.clone(); if let Some(existing_symlinks) = existing_symlinks { @@ -113,6 +115,11 @@ impl Locator for MacCmdLineTools { // Use the latest accurate information we have. version = Some(resolved_env.version); prefix = Some(resolved_env.prefix); + arch = if resolved_env.is64_bit { + Some(Architecture::X64) + } else { + Some(Architecture::X86) + }; } } } @@ -165,11 +172,17 @@ impl Locator for MacCmdLineTools { version = version::from_header_files(prefix); } } + if version.is_none() || prefix.is_none() { if let Some(resolved_env) = ResolvedPythonEnv::from(&env.executable) { resolved_environments.push(resolved_env.clone()); version = Some(resolved_env.version); prefix = Some(resolved_env.prefix); + arch = if resolved_env.is64_bit { + Some(Architecture::X64) + } else { + Some(Architecture::X86) + }; } } @@ -177,6 +190,7 @@ impl Locator for MacCmdLineTools { .executable(Some(env.executable.clone())) .version(version) .prefix(prefix) + .arch(arch) .symlinks(Some(symlinks.clone())) .build(); diff --git a/crates/pet-mac-python-org/src/lib.rs b/crates/pet-mac-python-org/src/lib.rs index e9573d5c..73642ef1 100644 --- a/crates/pet-mac-python-org/src/lib.rs +++ b/crates/pet-mac-python-org/src/lib.rs @@ -5,7 +5,7 @@ use pet_core::{ env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_fs::path::resolve_symlink; use pet_python_utils::executable::find_executables; @@ -27,8 +27,8 @@ impl Default for MacPythonOrg { } } impl Locator for MacPythonOrg { - fn get_name(&self) -> &'static str { - "MacPythonOrg" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::MacPythonOrg } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::MacPythonOrg] diff --git a/crates/pet-mac-xcode/src/lib.rs b/crates/pet-mac-xcode/src/lib.rs index 05f36090..ab86adeb 100644 --- a/crates/pet-mac-xcode/src/lib.rs +++ b/crates/pet-mac-xcode/src/lib.rs @@ -2,10 +2,11 @@ // Licensed under the MIT License. use pet_core::{ + arch::Architecture, env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_fs::path::resolve_symlink; use pet_python_utils::version; @@ -26,8 +27,8 @@ impl Default for MacXCode { } } impl Locator for MacXCode { - fn get_name(&self) -> &'static str { - "MacXCode" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::MacXCode } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::MacCommandLineTools] @@ -56,6 +57,7 @@ impl Locator for MacXCode { let mut version = env.version.clone(); let mut prefix = env.prefix.clone(); let mut symlinks = vec![env.executable.clone()]; + let mut arch = None; let existing_symlinks = env.symlinks.clone(); if let Some(existing_symlinks) = existing_symlinks { @@ -110,6 +112,11 @@ impl Locator for MacXCode { // Use the latest accurate information we have. version = Some(resolved_env.version); prefix = Some(resolved_env.prefix); + arch = if resolved_env.is64_bit { + Some(Architecture::X64) + } else { + Some(Architecture::X86) + }; } } } @@ -156,6 +163,11 @@ impl Locator for MacXCode { resolved_environments.push(resolved_env.clone()); version = Some(resolved_env.version); prefix = Some(resolved_env.prefix); + arch = if resolved_env.is64_bit { + Some(Architecture::X64) + } else { + Some(Architecture::X86) + }; } } @@ -163,6 +175,7 @@ impl Locator for MacXCode { .executable(Some(env.executable.clone())) .version(version) .prefix(prefix) + .arch(arch) .symlinks(Some(symlinks)) .build(); diff --git a/crates/pet-pipenv/src/lib.rs b/crates/pet-pipenv/src/lib.rs index f64e2beb..b727a11d 100644 --- a/crates/pet-pipenv/src/lib.rs +++ b/crates/pet-pipenv/src/lib.rs @@ -4,6 +4,7 @@ use env_variables::EnvVariables; use pet_core::env::PythonEnv; use pet_core::os_environment::Environment; +use pet_core::LocatorKind; use pet_core::{ python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, @@ -71,8 +72,8 @@ impl PipEnv { } } impl Locator for PipEnv { - fn get_name(&self) -> &'static str { - "PipEnv" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::PipEnv } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::Pipenv] diff --git a/crates/pet-poetry/src/lib.rs b/crates/pet-poetry/src/lib.rs index 85f6f215..30960ae1 100644 --- a/crates/pet-poetry/src/lib.rs +++ b/crates/pet-poetry/src/lib.rs @@ -10,7 +10,7 @@ use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, - Configuration, Locator, LocatorResult, + Configuration, Locator, LocatorKind, LocatorResult, }; use pet_virtualenv::is_virtualenv; use std::{ @@ -127,8 +127,8 @@ impl PoetryLocator for Poetry { } impl Locator for Poetry { - fn get_name(&self) -> &'static str { - "Poetry" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::Poetry } fn configure(&self, config: &Configuration) { if let Some(workspace_directories) = &config.workspace_directories { diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index 741e89d6..a8d152ef 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -19,7 +19,7 @@ use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_python_utils::executable::find_executable; @@ -75,13 +75,14 @@ impl PyEnv { } impl Locator for PyEnv { - fn get_name(&self) -> &'static str { - "PyEnv" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::PyEnv } fn supported_categories(&self) -> Vec { vec![ PythonEnvironmentKind::Pyenv, PythonEnvironmentKind::PyenvVirtualEnv, + PythonEnvironmentKind::Conda, ] } diff --git a/crates/pet-python-utils/src/env.rs b/crates/pet-python-utils/src/env.rs index 842fb02c..15a21a55 100644 --- a/crates/pet-python-utils/src/env.rs +++ b/crates/pet-python-utils/src/env.rs @@ -59,7 +59,10 @@ impl ResolvedPythonEnv { let entry = cache.lock().unwrap(); entry.track_symlinks(symlinks) } else { - error!("Invalid Python environment being cached: {:?}", environment); + error!( + "Invalid Python environment being cached: {:?} expected {:?}", + environment, self + ); } } /// Given the executable path, resolve the python environment by spawning python. diff --git a/crates/pet-python-utils/src/fs_cache.rs b/crates/pet-python-utils/src/fs_cache.rs index 7de515d0..e53c75ad 100644 --- a/crates/pet-python-utils/src/fs_cache.rs +++ b/crates/pet-python-utils/src/fs_cache.rs @@ -24,7 +24,7 @@ struct CacheEntry { } pub fn generate_cache_file(cache_directory: &Path, executable: &PathBuf) -> PathBuf { - cache_directory.join(format!("{}.2.json", generate_hash(executable))) + cache_directory.join(format!("{}.3.json", generate_hash(executable))) } pub fn delete_cache_file(cache_directory: &Path, executable: &PathBuf) { diff --git a/crates/pet-reporter/src/jsonrpc.rs b/crates/pet-reporter/src/jsonrpc.rs index 7842ae23..5642a438 100644 --- a/crates/pet-reporter/src/jsonrpc.rs +++ b/crates/pet-reporter/src/jsonrpc.rs @@ -5,14 +5,16 @@ use env_logger::Builder; use log::{trace, LevelFilter}; use pet_core::{ manager::EnvManager, - python_environment::PythonEnvironment, + python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, telemetry::{get_telemetry_event_name, TelemetryEvent}, }; use pet_jsonrpc::send_message; use serde::{Deserialize, Serialize}; -pub struct JsonRpcReporter {} +pub struct JsonRpcReporter { + report_only: Option, +} #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -37,13 +39,26 @@ impl Reporter for JsonRpcReporter { } fn report_environment(&self, env: &PythonEnvironment) { + if let Some(report_only) = &self.report_only { + if env.kind != Some(*report_only) { + trace!( + "Skip Reporting Environment ({:?}) {:?} due to refresh request to report only {:?}", + env.kind, + env.executable + .clone() + .unwrap_or(env.prefix.clone().unwrap_or_default()), + report_only + ); + return; + } + } trace!("Reporting Environment {:?}", env); send_message("environment", env.into()) } } -pub fn create_reporter() -> impl Reporter { - JsonRpcReporter {} +pub fn create_reporter(report_only: Option) -> impl Reporter { + JsonRpcReporter { report_only } } #[derive(Serialize, Deserialize, PartialEq, Debug, Eq, Clone)] diff --git a/crates/pet-reporter/src/stdio.rs b/crates/pet-reporter/src/stdio.rs index 85bd7dc4..8cf9d1ca 100644 --- a/crates/pet-reporter/src/stdio.rs +++ b/crates/pet-reporter/src/stdio.rs @@ -18,6 +18,7 @@ pub struct StdioReporter { print_list: bool, managers: Arc>>, environments: Arc, u16>>>, + kind: Option, } pub struct Summary { @@ -49,6 +50,9 @@ impl Reporter for StdioReporter { } fn report_environment(&self, env: &PythonEnvironment) { + if self.kind.is_some() && env.kind != self.kind { + return; + } let mut environments = self.environments.lock().unwrap(); let count = environments.get(&env.kind).unwrap_or(&0) + 1; environments.insert(env.kind, count); @@ -58,11 +62,12 @@ impl Reporter for StdioReporter { } } -pub fn create_reporter(print_list: bool) -> StdioReporter { +pub fn create_reporter(print_list: bool, kind: Option) -> StdioReporter { StdioReporter { print_list, managers: Arc::new(Mutex::new(HashMap::new())), environments: Arc::new(Mutex::new(HashMap::new())), + kind, } } diff --git a/crates/pet-venv/src/lib.rs b/crates/pet-venv/src/lib.rs index a471914a..d9b9caac 100644 --- a/crates/pet-venv/src/lib.rs +++ b/crates/pet-venv/src/lib.rs @@ -8,7 +8,7 @@ use pet_core::{ python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, pyvenv_cfg::PyVenvCfg, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_python_utils::executable::find_executables; use pet_python_utils::version; @@ -39,8 +39,8 @@ impl Default for Venv { } } impl Locator for Venv { - fn get_name(&self) -> &'static str { - "Venv" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::Venv } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::Venv] diff --git a/crates/pet-virtualenv/src/lib.rs b/crates/pet-virtualenv/src/lib.rs index c14dab1c..82d5894c 100644 --- a/crates/pet-virtualenv/src/lib.rs +++ b/crates/pet-virtualenv/src/lib.rs @@ -7,7 +7,7 @@ use pet_core::{ env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_python_utils::executable::find_executables; use pet_python_utils::version; @@ -86,8 +86,8 @@ impl Default for VirtualEnv { } impl Locator for VirtualEnv { - fn get_name(&self) -> &'static str { - "VirtualEnv" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::VirtualEnv } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::VirtualEnv] diff --git a/crates/pet-virtualenvwrapper/src/lib.rs b/crates/pet-virtualenvwrapper/src/lib.rs index 335c99ed..4d4a5dce 100644 --- a/crates/pet-virtualenvwrapper/src/lib.rs +++ b/crates/pet-virtualenvwrapper/src/lib.rs @@ -8,7 +8,7 @@ use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, - Locator, + Locator, LocatorKind, }; use pet_python_utils::executable::find_executables; use pet_python_utils::version; @@ -30,8 +30,8 @@ impl VirtualEnvWrapper { } impl Locator for VirtualEnvWrapper { - fn get_name(&self) -> &'static str { - "VirtualEnvWrapper" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::VirtualEnvWrapper } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::VirtualEnvWrapper] diff --git a/crates/pet-windows-registry/src/lib.rs b/crates/pet-windows-registry/src/lib.rs index 9e8afcfc..29dfa7fe 100644 --- a/crates/pet-windows-registry/src/lib.rs +++ b/crates/pet-windows-registry/src/lib.rs @@ -8,7 +8,7 @@ use pet_core::{ env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, - Locator, LocatorResult, + Locator, LocatorKind, LocatorResult, }; use pet_virtualenv::is_virtualenv; use std::sync::{Arc, Mutex}; @@ -49,11 +49,14 @@ impl WindowsRegistry { } impl Locator for WindowsRegistry { - fn get_name(&self) -> &'static str { - "WindowsRegistry" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::WindowsRegistry } fn supported_categories(&self) -> Vec { - vec![PythonEnvironmentKind::WindowsRegistry] + vec![ + PythonEnvironmentKind::WindowsRegistry, + PythonEnvironmentKind::Conda, + ] } fn try_from(&self, env: &PythonEnv) -> Option { diff --git a/crates/pet-windows-store/src/lib.rs b/crates/pet-windows-store/src/lib.rs index 6e507961..68ec03b7 100644 --- a/crates/pet-windows-store/src/lib.rs +++ b/crates/pet-windows-store/src/lib.rs @@ -11,6 +11,7 @@ use environments::list_store_pythons; use pet_core::env::PythonEnv; use pet_core::python_environment::{PythonEnvironment, PythonEnvironmentKind}; use pet_core::reporter::Reporter; +use pet_core::LocatorKind; use pet_core::{os_environment::Environment, Locator}; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -51,8 +52,8 @@ impl WindowsStore { } impl Locator for WindowsStore { - fn get_name(&self) -> &'static str { - "WindowsStore" // Do not change this name, as this is used in telemetry. + fn get_kind(&self) -> LocatorKind { + LocatorKind::WindowsStore } fn supported_categories(&self) -> Vec { vec![PythonEnvironmentKind::WindowsStore] diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs index 6b0bdfc8..3b835281 100644 --- a/crates/pet/src/find.rs +++ b/crates/pet/src/find.rs @@ -5,8 +5,9 @@ use log::{trace, warn}; use pet_conda::utils::is_conda_env; use pet_core::env::PythonEnv; use pet_core::os_environment::Environment; +use pet_core::python_environment::PythonEnvironmentKind; use pet_core::reporter::Reporter; -use pet_core::{Configuration, Locator}; +use pet_core::{Configuration, Locator, LocatorKind}; use pet_env_var_path::get_search_paths_from_env_variables; use pet_global_virtualenvs::list_global_virtual_envs_paths; use pet_python_utils::executable::{ @@ -25,7 +26,7 @@ use crate::locators::identify_python_environment_using_locators; pub struct Summary { pub total: Duration, - pub locators: BTreeMap<&'static str, Duration>, + pub locators: BTreeMap, pub breakdown: BTreeMap<&'static str, Duration>, } @@ -33,7 +34,7 @@ pub struct Summary { #[serde(rename_all = "camelCase")] pub enum SearchScope { /// Search for environments in global space. - Global, + Global(PythonEnvironmentKind), /// Search for environments in workspace folder. Workspace, } @@ -55,15 +56,15 @@ pub fn find_and_report_envs( // From settings let environment_directories = configuration.environment_directories.unwrap_or_default(); let workspace_directories = configuration.workspace_directories.unwrap_or_default(); + let executables = configuration.executables.unwrap_or_default(); let search_global = match search_scope { - Some(SearchScope::Global) => true, + Some(SearchScope::Global(_)) => true, Some(SearchScope::Workspace) => false, _ => true, }; - let search_workspace = match search_scope { - Some(SearchScope::Global) => false, - Some(SearchScope::Workspace) => true, - _ => true, + let search_kind = match search_scope { + Some(SearchScope::Global(kind)) => Some(kind), + _ => None, }; thread::scope(|s| { @@ -74,22 +75,33 @@ pub fn find_and_report_envs( if search_global { thread::scope(|s| { for locator in locators.iter() { + if let Some(kind) = &search_kind { + if !locator.supported_categories().contains(kind) { + trace!( + "Skipping locator: {:?} as it does not support {:?} (required by refresh command)", + locator.get_kind(), + kind + ); + continue; + } + } + let locator = locator.clone(); let summary = summary.clone(); s.spawn(move || { let start = std::time::Instant::now(); - trace!("Searching using locator: {}", locator.get_name()); + trace!("Searching using locator: {:?}", locator.get_kind()); locator.find(reporter); trace!( - "Completed searching using locator: {} in {:?}", - locator.get_name(), + "Completed searching using locator: {:?} in {:?}", + locator.get_kind(), start.elapsed() ); summary .lock() .unwrap() .locators - .insert(locator.get_name(), start.elapsed()); + .insert(locator.get_kind(), start.elapsed()); }); } }); @@ -169,25 +181,41 @@ pub fn find_and_report_envs( // that could the discovery. s.spawn(|| { let start = std::time::Instant::now(); - if search_workspace && !workspace_directories.is_empty() { - trace!( - "Searching for environments in workspace folders: {:?}", - workspace_directories - ); - let global_env_search_paths: Vec = - get_search_paths_from_env_variables(environment); - for workspace_folder in workspace_directories { - let global_env_search_paths = global_env_search_paths.clone(); - s.spawn(move || { - find_python_environments_in_workspace_folder_recursive( - &workspace_folder, - reporter, - locators, - &global_env_search_paths, - ); - }); + thread::scope(|s| { + // Find environments in the workspace folders. + if !workspace_directories.is_empty() { + trace!( + "Searching for environments in workspace folders: {:?}", + workspace_directories + ); + let global_env_search_paths: Vec = + get_search_paths_from_env_variables(environment); + for workspace_folder in workspace_directories { + let global_env_search_paths = global_env_search_paths.clone(); + s.spawn(move || { + find_python_environments_in_workspace_folder_recursive( + &workspace_folder, + reporter, + locators, + &global_env_search_paths, + ); + }); + } } - } + // Find the python exes provided. + if !executables.is_empty() { + trace!("Searching for environment executables: {:?}", executables); + let global_env_search_paths: Vec = + get_search_paths_from_env_variables(environment); + identify_python_executables_using_locators( + executables, + locators, + reporter, + &global_env_search_paths, + ); + } + }); + summary .lock() .unwrap() diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index d4d24f50..d3072f0b 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -12,6 +12,7 @@ use pet::resolve::resolve_environment; use pet_conda::Conda; use pet_conda::CondaLocator; use pet_core::python_environment::PythonEnvironment; +use pet_core::python_environment::PythonEnvironmentKind; use pet_core::telemetry::refresh_performance::RefreshPerformance; use pet_core::telemetry::TelemetryEvent; use pet_core::{ @@ -136,9 +137,12 @@ pub fn handle_configure(context: Arc, id: u32, params: Value) { #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct RefreshOptions { - /// The search paths are the paths where we will look for environments. - /// Defaults to searching everywhere (or when None), else it can be restricted to a specific scope. - pub search_scope: Option, + /// If provided, then limit the search to this kind of environments. + pub search_kind: Option, + /// If provided, then limit the search paths to these. + /// Note: Search paths can also include Python exes or Python env folders. + /// Traditionally, search paths are workspace folders. + pub search_paths: Option>, } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -160,14 +164,47 @@ pub fn handle_refresh(context: Arc, id: u32, params: Value) { _ => params, }; match serde_json::from_value::(params.clone()) { - Ok(refres_options) => { + Ok(refresh_options) => { // Start in a new thread, we can have multiple requests. thread::spawn(move || { // Ensure we can have only one refresh at a time. let lock = REFRESH_LOCK.lock().unwrap(); - let config = context.configuration.read().unwrap().clone(); - let reporter = Arc::new(CacheReporter::new(Arc::new(jsonrpc::create_reporter()))); + let mut config = context.configuration.read().unwrap().clone(); + let reporter = Arc::new(CacheReporter::new(Arc::new(jsonrpc::create_reporter( + refresh_options.search_kind, + )))); + + let mut search_scope = None; + + // If search kind is provided and no search_paths, then we will only search in the global locations. + config.executables = None; // This can only be provided in the refresh request. + if refresh_options.search_kind.is_some() || refresh_options.search_paths.is_some() { + config.workspace_directories = None; + if let Some(search_paths) = refresh_options.search_paths { + config.workspace_directories = Some( + search_paths + .iter() + .filter(|p| p.is_dir()) + .cloned() + .collect(), + ); + config.executables = Some( + search_paths + .iter() + .filter(|p| p.is_file()) + .cloned() + .collect(), + ); + search_scope = Some(SearchScope::Workspace); + } else if let Some(search_kind) = refresh_options.search_kind { + config.executables = None; + search_scope = Some(SearchScope::Global(search_kind)); + } + for locator in context.locators.iter() { + locator.configure(&config); + } + } trace!("Start refreshing environments, config: {:?}", config); let summary = find_and_report_envs( @@ -175,11 +212,11 @@ pub fn handle_refresh(context: Arc, id: u32, params: Value) { config, &context.locators, context.os_environment.deref(), - refres_options.search_scope, + search_scope, ); let summary = summary.lock().unwrap(); for locator in summary.locators.iter() { - info!("Locator {} took {:?}", locator.0, locator.1); + info!("Locator {:?} took {:?}", locator.0, locator.1); } for item in summary.breakdown.iter() { info!("Locator {} took {:?}", item.0, item.1); @@ -193,7 +230,7 @@ pub fn handle_refresh(context: Arc, id: u32, params: Value) { .locators .clone() .iter() - .map(|(k, v)| (k.to_string(), v.as_millis())) + .map(|(k, v)| (format!("{:?}", k), v.as_millis())) .collect::>(), breakdown: summary .breakdown @@ -279,7 +316,7 @@ pub fn handle_resolve(context: Arc, id: u32, params: Value) { { if let Some(resolved) = result.resolved { // Gather telemetry of this resolved env and see what we got wrong. - let jsonrpc_reporter = jsonrpc::create_reporter(); + let jsonrpc_reporter = jsonrpc::create_reporter(None); let _ = report_inaccuracies_identified_after_resolving( &jsonrpc_reporter, &result.discovered, diff --git a/crates/pet/src/lib.rs b/crates/pet/src/lib.rs index 3f31accd..735d36ab 100644 --- a/crates/pet/src/lib.rs +++ b/crates/pet/src/lib.rs @@ -2,20 +2,17 @@ // Licensed under the MIT License. use find::find_and_report_envs; -use find::identify_python_executables_using_locators; use find::SearchScope; use locators::create_locators; -use log::warn; use pet_conda::Conda; use pet_conda::CondaLocator; use pet_core::os_environment::Environment; +use pet_core::python_environment::PythonEnvironmentKind; use pet_core::Locator; use pet_core::{os_environment::EnvironmentApi, reporter::Reporter, Configuration}; -use pet_env_var_path::get_search_paths_from_env_variables; use pet_poetry::Poetry; use pet_poetry::PoetryLocator; use pet_python_utils::cache::set_cache_directory; -use pet_reporter::collect; use pet_reporter::{self, cache::CacheReporter, stdio}; use resolve::resolve_environment; use std::path::PathBuf; @@ -31,10 +28,10 @@ pub struct FindOptions { pub print_summary: bool, pub verbose: bool, pub report_missing: bool, - pub workspace_dirs: Option>, + pub search_paths: Option>, pub workspace_only: bool, - pub global_only: bool, pub cache_directory: Option, + pub kind: Option, } pub fn find_and_report_envs_stdio(options: FindOptions) { @@ -44,18 +41,16 @@ pub fn find_and_report_envs_stdio(options: FindOptions) { log::LevelFilter::Warn }); let now = SystemTime::now(); + let config = create_config(&options); let search_scope = if options.workspace_only { Some(SearchScope::Workspace) - } else if options.global_only { - Some(SearchScope::Global) } else { - None + options.kind.map(SearchScope::Global) }; if let Some(cache_directory) = options.cache_directory.clone() { set_cache_directory(cache_directory); } - let (config, executable_to_find) = create_config(&options); let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); let poetry_locator = Arc::new(Poetry::from(&environment)); @@ -65,47 +60,51 @@ pub fn find_and_report_envs_stdio(options: FindOptions) { locator.configure(&config); } - if let Some(executable) = executable_to_find { - find_env(&executable, &locators, &environment) - } else { - find_envs( - &options, - &locators, - config, - conda_locator.as_ref(), - poetry_locator.as_ref(), - &environment, - search_scope, - ); - } + find_envs( + &options, + &locators, + config, + conda_locator.as_ref(), + poetry_locator.as_ref(), + &environment, + search_scope, + ); println!("Completed in {}ms", now.elapsed().unwrap().as_millis()) } -fn create_config(options: &FindOptions) -> (Configuration, Option) { +fn create_config(options: &FindOptions) -> Configuration { let mut config = Configuration::default(); - let mut workspace_directories = vec![]; - if let Some(dirs) = options.workspace_dirs.clone() { - workspace_directories.extend(dirs); + + let mut search_paths = vec![]; + if let Some(dirs) = options.search_paths.clone() { + search_paths.extend(dirs); } // If workspace folders have been provided do not add cwd. - if workspace_directories.is_empty() { + if search_paths.is_empty() { if let Ok(cwd) = env::current_dir() { - workspace_directories.push(cwd); + search_paths.push(cwd); } } - workspace_directories.sort(); - workspace_directories.dedup(); - - let executable_to_find = - if workspace_directories.len() == 1 && workspace_directories[0].is_file() { - Some(workspace_directories[0].clone()) - } else { - None - }; - config.workspace_directories = Some(workspace_directories); + search_paths.sort(); + search_paths.dedup(); + + config.workspace_directories = Some( + search_paths + .iter() + .filter(|d| d.is_dir()) + .cloned() + .collect(), + ); + config.executables = Some( + search_paths + .iter() + .filter(|d| d.is_file()) + .cloned() + .collect(), + ); - (config, executable_to_find) + config } fn find_envs( @@ -117,7 +116,11 @@ fn find_envs( environment: &dyn Environment, search_scope: Option, ) { - let stdio_reporter = Arc::new(stdio::create_reporter(options.print_list)); + let kind = match search_scope { + Some(SearchScope::Global(kind)) => Some(kind), + _ => None, + }; + let stdio_reporter = Arc::new(stdio::create_reporter(options.print_list, kind)); let reporter = CacheReporter::new(stdio_reporter.clone()); let summary = find_and_report_envs(&reporter, config, locators, environment, search_scope); @@ -136,7 +139,7 @@ fn find_envs( println!("Breakdown by each locator:"); println!("--------------------------"); for locator in summary.locators.iter() { - println!("{:<20} : {:?}", locator.0, locator.1); + println!("{:<20} : {:?}", format!("{:?}", locator.0), locator.1); } println!() } @@ -193,39 +196,6 @@ fn find_envs( } } -fn find_env( - executable: &PathBuf, - locators: &Arc>>, - environment: &dyn Environment, -) { - let collect_reporter = Arc::new(collect::create_reporter()); - let reporter = CacheReporter::new(collect_reporter.clone()); - let stdio_reporter = Arc::new(stdio::create_reporter(true)); - - let global_env_search_paths: Vec = get_search_paths_from_env_variables(environment); - - identify_python_executables_using_locators( - vec![executable.clone()], - locators, - &reporter, - &global_env_search_paths, - ); - - // Find the environment for the file provided. - let environments = collect_reporter.environments.lock().unwrap(); - if let Some(env) = environments - .iter() - .find(|e| e.symlinks.clone().unwrap_or_default().contains(executable)) - { - if let Some(manager) = &env.manager { - stdio_reporter.report_manager(manager); - } - stdio_reporter.report_environment(env); - } else { - warn!("Failed to find the environment for {:?}", executable); - } -} - pub fn resolve_report_stdio(executable: PathBuf, verbose: bool, cache_directory: Option) { stdio::initialize_logger(if verbose { log::LevelFilter::Trace @@ -238,7 +208,7 @@ pub fn resolve_report_stdio(executable: PathBuf, verbose: bool, cache_directory: set_cache_directory(cache_directory); } - let stdio_reporter = Arc::new(stdio::create_reporter(true)); + let stdio_reporter = Arc::new(stdio::create_reporter(true, None)); let reporter = CacheReporter::new(stdio_reporter.clone()); let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); diff --git a/crates/pet/src/main.rs b/crates/pet/src/main.rs index f8eee2ad..f22bf369 100644 --- a/crates/pet/src/main.rs +++ b/crates/pet/src/main.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use clap::{Parser, Subcommand}; use jsonrpc::start_jsonrpc_server; use pet::{find_and_report_envs_stdio, resolve_report_stdio, FindOptions}; +use pet_core::python_environment::PythonEnvironmentKind; mod find; mod jsonrpc; @@ -22,10 +23,10 @@ pub struct Cli { enum Commands { /// Finds the environments and reports them to the standard output. Find { - /// List of folders to search for environments. + /// List of files/folders to search for environments. /// The current directory is automatically used as a workspace folder if none provided. - #[arg(value_name = "WORKSPACE FOLDERS")] - workspace_dirs: Option>, + #[arg(value_name = "SEARCH PATHS")] + search_paths: Option>, /// List the environments found. #[arg(short, long)] @@ -45,13 +46,13 @@ enum Commands { /// Exclusively search just the workspace directories. /// I.e. exclude all global environments. - #[arg(short, long, conflicts_with = "global_only")] - workspace_only: bool, + #[arg(short, long, conflicts_with = "kind")] + workspace: bool, - /// Exclusively search just the global environments (Conda, Poetry, Registry, etc). - /// I.e. do not search the workspace directories. - #[arg(short, long, conflicts_with = "workspace_only")] - global_only: bool, + /// Exclusively search for a specific Python environment kind. + /// Will not search in the workspace directories. + #[arg(short, long, conflicts_with = "workspace")] + kind: Option, }, /// Resolves & reports the details of the the environment to the standard output. Resolve { @@ -78,29 +79,42 @@ fn main() { list: true, verbose: false, report_missing: false, - workspace_dirs: None, - workspace_only: false, - global_only: false, + search_paths: None, + workspace: false, cache_directory: None, + kind: None, }) { Commands::Find { list, verbose, report_missing, - workspace_dirs, - workspace_only, - global_only, + search_paths, + workspace, cache_directory, - } => find_and_report_envs_stdio(FindOptions { - print_list: list, - print_summary: true, - verbose, - report_missing, - workspace_dirs, - workspace_only, - global_only, - cache_directory, - }), + kind, + } => { + let mut workspace_only = workspace; + if search_paths.clone().is_some() + && search_paths + .clone() + .unwrap_or_default() + .iter() + .all(|f| f.is_file()) + { + workspace_only = true; + } + + find_and_report_envs_stdio(FindOptions { + print_list: list, + print_summary: true, + verbose, + report_missing, + search_paths, + workspace_only, + cache_directory, + kind, + }); + } Commands::Resolve { executable, verbose, diff --git a/docs/JSONRPC.md b/docs/JSONRPC.md index c98ad23a..31006e48 100644 --- a/docs/JSONRPC.md +++ b/docs/JSONRPC.md @@ -86,13 +86,17 @@ _Response_: ```typescript interface RefreshParams { /** - * Determines the scope of Python environments to search for. - * If not provided, then the tool will search for Python environments in the global & workspace scope. - * - * 'global' - Search for Python environments in the global scope (useful to search for new Conda environments created in usual `envs` directory). - * 'workspace' - Search for Python environments in the workspace scope (useful to search for new virtual environments or conda environments created in the workspace). + * Limits the search to a specific kind of Python environment. + * Ignores workspace folders passed in configuration request. + */ + searchKind?: PythonEnvironmentKind; +} | { + /** + * Limits the search to a specific set of paths. + * searchPaths can either by directories or Python prefixes/executables or combination of both. + * Ignores workspace folders passed in configuration request. */ - searchScopr?: "global" | "workspace"; + searchPaths?: string[]; } interface RefreshResult { @@ -137,6 +141,25 @@ interface ResolveParams { executable: string; } +enum PythonEnvironmentKind { + Conda, + Homebrew, + Pyenv, + GlobalPaths, // Python found in global locations like PATH, /usr/bin etc. + PyenvVirtualEnv, // Pyenv virtualenvs. + Pipenv, + Poetry, + MacPythonOrg, // Python installed from python.org on Mac + MacCommandLineTools, + LinuxGlobal, // Python installed in Linux in paths such as `/usr/bin`, `/usr/local/bin` etc. + MacXCode, + Venv, + VirtualEnv, + VirtualEnvWrapper, + WindowsStore, + WindowsRegistry, +} + interface Environment { /** * The display name of the enviornment. @@ -163,23 +186,7 @@ interface Environment { /** * The kind of the environment. */ - kind?: - | "Conda" // Conda environment - | "GlobalPaths" // Unknown Pyton environment, found in the PATH environment variable - | "Homebrew" // Homebrew installed Python - | "LinuxGlobal" // Python installed from the system package manager on Linux - | "MacCommandLineTools" // Python installed from the Mac command line tools - | "MacPythonOrg" // Python installed from python.org on Mac - | "MacXCode" // Python installed from XCode on Mac - | "Pipenv" // Pipenv environment - | "Poetry" // Poetry environment - | "Pyenv" // Pyenv installed Python - | "PyenvVirtualEnv" // pyenv-virtualenv environment - | "Venv" // Python venv environment (generally created using the `venv` module) - | "VirtualEnv" // Python virtual environment - | "VirtualEnvWrapper" // Virtualenvwrapper Environment - | "WindowsRegistry" // Python installed & found in Windows Registry - | "WindowsStore"; // Python installed from the Windows Store + kind?: PythonEnvironmentKind; /** * The version of the python executable. * This will at a minimum contain the 3 parts of the version such as `3.8.1`. @@ -239,54 +246,6 @@ interface Manager { } ``` -# Find Request - -Use this request to find Python environments in a specific directory. - -**Notes:** - -- This request will also search for Python environments in other directories that have been configured for use with the specified folder. - E.g. Poetry, Pipenv, Virtualenvwrapper, etc environments associated with the provided directory will be returned. -- If the directory such as `/.venv/bin` is passed, then the tool will return the Python environment associated with the same environment. - I.e. in this case the returned environment will point to `/.venv/bin/python`. - This is similar to the `refresh` request, but limited to returning just the Python environment associated with the provided path. -- If the file such as `/.venv/bin/python` is passed, then the tool will return the Python environment associated with the same environment. - I.e. in this case the returned environment will point to `/.venv/bin/python`. - This is similar to the `refresh` request, but limited to returning just the Python environment associated with the provided path. - -_Why use this over the `refresh` request?_ - -This will search for Python environments in the provided directory and can be faster than searching for Python environments in the global & workspace scope. -Or if you're only interested in Python environments in a specific directory, that might not be a workspace directory. - -_How is this different from `resolve` request when passing the `?_ - -With `resolve`, Python executable is spawned and can be slow, when all that is required is to get some basic information about the Python environment. -E.g. if the Python executable `/.venv/bin/python` for the virtual environment is passed to this request, then the tool will return the Python environment associated with the same environment. E.g. it will indicate that this is a virtual environment and return the version, prefix, etc. -However passing the same information to the `resolve` request will spawn the Python executable and return some additional information, such as whether its `64bit` or `32bit`, but will be slower. - -_Request_: - -- method: `find` -- params: `FindParams` defined as below. - -_Response_: - -- result: `Environment` defined as earlier. - -```typescript -interface FindParams { - /** - * The fully qualified path to the directory to search for Python environments. - * Or could also be the path to a Python environment. - * Or even the path to a Python executable. - * - * E.g. `/.venv` or `/.venv/bin` or `/.venv/bin/python` or some other path that even contains the workspace folders. - */ - searchPath: string; -} -``` - # Clear Cache Request Use this request to clear the cache that the tool uses to store Python environment details. From 7000221162bd5b4f78d20384d5635a22c9aaf094 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 18 Jul 2024 15:03:25 +1000 Subject: [PATCH 2/4] wip --- crates/pet-conda/src/environment_locations.rs | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/crates/pet-conda/src/environment_locations.rs b/crates/pet-conda/src/environment_locations.rs index a059de75..6dcd2bd2 100644 --- a/crates/pet-conda/src/environment_locations.rs +++ b/crates/pet-conda/src/environment_locations.rs @@ -13,6 +13,7 @@ use std::{ env, fs, path::{Path, PathBuf}, thread, + time::SystemTime, }; const APP_NAME: &str = "conda"; @@ -21,13 +22,14 @@ pub fn get_conda_environment_paths( env_vars: &EnvVariables, conda_executable: &Option, ) -> Vec { + let start = SystemTime::now(); let mut env_paths = thread::scope(|s| { let mut envs = vec![]; for thread in [ s.spawn(|| get_conda_envs_from_environment_txt(env_vars)), s.spawn(|| get_conda_environment_paths_from_conda_rc(env_vars)), - s.spawn(|| get_conda_environment_paths_from_known_paths(env_vars)), - s.spawn(|| get_known_conda_install_locations(env_vars, conda_executable)), + // s.spawn(|| get_conda_environment_paths_from_known_paths(env_vars)), + // s.spawn(|| get_known_conda_install_locations(env_vars, conda_executable)), ] { if let Ok(mut env_paths) = thread.join() { envs.append(&mut env_paths); @@ -43,7 +45,8 @@ pub fn get_conda_environment_paths( // & then iterate through the list of envs in the envs directory. // let env_paths = vec![]; let mut threads = vec![]; - for path in env_paths { + println!("Env Paths: {:?}", env_paths); + for path in env_paths.iter().filter(|f| f.exists()) { let path = path.clone(); threads.push(thread::spawn(move || get_environments(&path))); } @@ -57,6 +60,11 @@ pub fn get_conda_environment_paths( result.sort(); result.dedup(); + println!( + "Time taken to get conda environments: {:?}", + start.elapsed().unwrap() + ); + println!("Paths: {:?}", result); result } @@ -68,13 +76,24 @@ fn get_conda_environment_paths_from_conda_rc(env_vars: &EnvVariables) -> Vec Vec Date: Thu, 18 Jul 2024 15:25:50 +1000 Subject: [PATCH 3/4] Fixes --- crates/pet-conda/src/environment_locations.rs | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/crates/pet-conda/src/environment_locations.rs b/crates/pet-conda/src/environment_locations.rs index 6dcd2bd2..bfaa575a 100644 --- a/crates/pet-conda/src/environment_locations.rs +++ b/crates/pet-conda/src/environment_locations.rs @@ -28,8 +28,8 @@ pub fn get_conda_environment_paths( for thread in [ s.spawn(|| get_conda_envs_from_environment_txt(env_vars)), s.spawn(|| get_conda_environment_paths_from_conda_rc(env_vars)), - // s.spawn(|| get_conda_environment_paths_from_known_paths(env_vars)), - // s.spawn(|| get_known_conda_install_locations(env_vars, conda_executable)), + s.spawn(|| get_conda_environment_paths_from_known_paths(env_vars)), + s.spawn(|| get_known_conda_install_locations(env_vars, conda_executable)), ] { if let Ok(mut env_paths) = thread.join() { envs.append(&mut env_paths); @@ -45,7 +45,6 @@ pub fn get_conda_environment_paths( // & then iterate through the list of envs in the envs directory. // let env_paths = vec![]; let mut threads = vec![]; - println!("Env Paths: {:?}", env_paths); for path in env_paths.iter().filter(|f| f.exists()) { let path = path.clone(); threads.push(thread::spawn(move || get_environments(&path))); @@ -60,11 +59,10 @@ pub fn get_conda_environment_paths( result.sort(); result.dedup(); - println!( - "Time taken to get conda environments: {:?}", + trace!( + "Time taken to get conda environment paths: {:?}", start.elapsed().unwrap() ); - println!("Paths: {:?}", result); result } @@ -170,6 +168,9 @@ fn get_conda_environment_paths_from_known_paths(env_vars: &EnvVariables) -> Vec< } } env_paths.append(&mut env_vars.known_global_search_locations.clone()); + env_paths.sort(); + env_paths.dedup(); + let env_paths = env_paths.into_iter().filter(|f| f.exists()).collect(); trace!("Conda environments in known paths {:?}", env_paths); env_paths } @@ -385,7 +386,7 @@ pub fn get_known_conda_install_locations( } if let Some(home) = env_vars.home.clone() { // https://stackoverflow.com/questions/35709497/anaconda-python-where-are-the-virtual-environments-stored - for prefix in [ + let mut prefixes = vec![ home.clone(), // https://towardsdatascience.com/manage-your-python-virtual-environment-with-conda-a0d2934d5195 home.join("opt"), @@ -396,9 +397,14 @@ pub fn get_known_conda_install_locations( PathBuf::from("/usr/share"), PathBuf::from("/usr/local"), PathBuf::from("/usr"), - PathBuf::from("/opt/homebrew"), - PathBuf::from("/home/linuxbrew/.linuxbrew"), - ] { + ]; + if std::env::consts::OS == "macos" { + prefixes.push(PathBuf::from("/opt/homebrew")); + } else { + prefixes.push(PathBuf::from("/home/linuxbrew/.linuxbrew")); + } + + for prefix in prefixes { known_paths.push(prefix.clone().join("anaconda")); known_paths.push(prefix.clone().join("anaconda3")); known_paths.push(prefix.clone().join("miniconda")); @@ -416,8 +422,7 @@ pub fn get_known_conda_install_locations( } known_paths.sort(); known_paths.dedup(); - - known_paths + known_paths.into_iter().filter(|f| f.exists()).collect() } pub fn get_conda_dir_from_exe(conda_executable: &Option) -> Option { From 5f6246a5685abd40c020f68ff4974569f4d1070b Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 18 Jul 2024 16:04:56 +1000 Subject: [PATCH 4/4] safe test to remove --- .../tests/environment_locations_test.rs | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/crates/pet-conda/tests/environment_locations_test.rs b/crates/pet-conda/tests/environment_locations_test.rs index 90bc4a53..c438d61d 100644 --- a/crates/pet-conda/tests/environment_locations_test.rs +++ b/crates/pet-conda/tests/environment_locations_test.rs @@ -3,52 +3,6 @@ mod common; -#[cfg(unix)] -#[test] -fn read_environment_txt() { - use common::{create_env_variables, resolve_test_path}; - use pet_conda::environment_locations::get_conda_envs_from_environment_txt; - use std::path::PathBuf; - - let root = resolve_test_path(&["unix", "root_empty"]).into(); - let home = resolve_test_path(&["unix", "user_home_with_environments_txt"]).into(); - let env = create_env_variables(home, root); - - let mut environments = get_conda_envs_from_environment_txt(&env); - environments.sort(); - - let mut expected = vec![ - "/Users/username/miniconda3", - "/Users/username/miniconda3/envs/xyz", - "/Users/username/miniconda3/envs/conda1", - "/Users/username/miniconda3/envs/conda2", - "/Users/username/miniconda3/envs/conda3", - "/Users/username/miniconda3/envs/conda4", - "/Users/username/miniconda3/envs/conda5", - "/Users/username/miniconda3/envs/conda6", - "/Users/username/miniconda3/envs/conda7", - "/Users/username/miniconda3/envs/conda8", - "/Users/username/.pyenv/versions/miniconda3-latest", - "/Users/username/.pyenv/versions/miniconda3-latest/envs/myenv", - "/Users/username/.pyenv/versions/miniforge3-4.10.1-1", - "/Users/username/.pyenv/versions/anaconda3-2023.03", - "/Users/username/miniforge3/envs/sample1", - "/Users/username/temp/conda_work_folder", - "/Users/username/temp/conda_work_folder_3.12", - "/Users/username/temp/conda_work_folder__no_python", - "/Users/username/temp/conda_work_folder_from_root", - "/Users/username/temp/sample-conda-envs-folder/hello_world", - "/Users/username/temp/sample-conda-envs-folder2/another", - "/Users/username/temp/sample-conda-envs-folder2/xyz", - ] - .iter() - .map(PathBuf::from) - .collect::>(); - expected.sort(); - - assert_eq!(environments, expected); -} - #[cfg(unix)] #[test] fn non_existent_envrionments_txt() {