From ea0cefaf36565072b47a39381a55a13a29072452 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 15 Jul 2024 17:01:22 +1000 Subject: [PATCH 1/3] wip --- Cargo.lock | 15 +- crates/pet-cache/Cargo.toml | 15 -- crates/pet-cache/src/lib.rs | 9 - crates/pet-conda/src/lib.rs | 1 + crates/pet-conda/tests/ci_test.rs | 6 +- crates/pet-conda/tests/lib_test.rs | 12 +- crates/pet-core/Cargo.toml | 1 - crates/pet-core/src/env.rs | 57 ++++++ crates/pet-core/src/lib.rs | 4 +- .../src/pyvenv_cfg.rs | 0 crates/pet-homebrew/src/lib.rs | 3 +- crates/pet-linux-global-python/src/lib.rs | 6 +- crates/pet-mac-commandlinetools/src/lib.rs | 6 +- crates/pet-mac-python-org/src/lib.rs | 2 +- crates/pet-mac-xcode/src/lib.rs | 6 +- crates/pet-pipenv/src/lib.rs | 2 +- crates/pet-poetry/src/lib.rs | 2 +- crates/pet-pyenv/src/lib.rs | 3 +- crates/pet-pyenv/tests/pyenv_test.rs | 2 +- crates/pet-python-utils/Cargo.toml | 1 + crates/pet-python-utils/src/cache.rs | 136 +++++++++++++ crates/pet-python-utils/src/env.rs | 184 +++++++++--------- crates/pet-python-utils/src/lib.rs | 17 +- crates/pet-python-utils/src/version.rs | 3 +- crates/pet-venv/src/lib.rs | 5 +- crates/pet-virtualenv/src/lib.rs | 3 +- .../pet-virtualenvwrapper/src/environments.rs | 2 +- crates/pet-virtualenvwrapper/src/lib.rs | 3 +- crates/pet-windows-registry/src/lib.rs | 2 +- crates/pet-windows-store/src/lib.rs | 2 +- crates/pet/src/find.rs | 2 +- crates/pet/src/jsonrpc.rs | 10 + crates/pet/src/locators.rs | 5 +- crates/pet/src/resolve.rs | 5 +- crates/pet/tests/ci_test.rs | 2 +- 35 files changed, 342 insertions(+), 192 deletions(-) delete mode 100644 crates/pet-cache/Cargo.toml delete mode 100644 crates/pet-cache/src/lib.rs create mode 100644 crates/pet-core/src/env.rs rename crates/{pet-python-utils => pet-core}/src/pyvenv_cfg.rs (100%) create mode 100644 crates/pet-python-utils/src/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 5b633317..2b436fac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,19 +369,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "pet-cache" -version = "0.1.0" -dependencies = [ - "log", - "msvc_spectre_libs", - "pet-core", - "pet-fs", - "pet-python-utils", - "serde", - "serde_json", -] - [[package]] name = "pet-conda" version = "0.1.0" @@ -408,7 +395,6 @@ dependencies = [ "log", "msvc_spectre_libs", "pet-fs", - "pet-python-utils", "regex", "serde", "serde_json", @@ -581,6 +567,7 @@ dependencies = [ "lazy_static", "log", "msvc_spectre_libs", + "pet-core", "pet-fs", "regex", "serde", diff --git a/crates/pet-cache/Cargo.toml b/crates/pet-cache/Cargo.toml deleted file mode 100644 index a37e838d..00000000 --- a/crates/pet-cache/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "pet-cache" -version = "0.1.0" -edition = "2021" - -[target.'cfg(target_os = "windows")'.dependencies] -msvc_spectre_libs = { version = "0.1.1", features = ["error"] } - -[dependencies] -pet-fs = { path = "../pet-fs" } -pet-python-utils = { path = "../pet-python-utils" } -serde = { version = "1.0.152", features = ["derive"] } -serde_json = "1.0.93" -pet-core = { path = "../pet-core" } -log = "0.4.21" diff --git a/crates/pet-cache/src/lib.rs b/crates/pet-cache/src/lib.rs deleted file mode 100644 index 95279817..00000000 --- a/crates/pet-cache/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use pet_core::python_environment::PythonEnvironment; - -pub trait Cache { - fn get>(&self, executable: P) -> Option; - fn set>(&self, environment: PythonEnvironment); -} diff --git a/crates/pet-conda/src/lib.rs b/crates/pet-conda/src/lib.rs index e9531508..ffc58ba9 100644 --- a/crates/pet-conda/src/lib.rs +++ b/crates/pet-conda/src/lib.rs @@ -11,6 +11,7 @@ use environments::{get_conda_environment_info, CondaEnvironment}; use log::error; use manager::CondaManager; use pet_core::{ + env::PythonEnv, os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, diff --git a/crates/pet-conda/tests/ci_test.rs b/crates/pet-conda/tests/ci_test.rs index 1ea75b4d..3722bd1f 100644 --- a/crates/pet-conda/tests/ci_test.rs +++ b/crates/pet-conda/tests/ci_test.rs @@ -69,10 +69,9 @@ fn detect_conda_root() { fn detect_conda_root_from_path() { use pet_conda::Conda; use pet_core::{ - manager::EnvManagerType, os_environment::EnvironmentApi, + env::PythonEnv, manager::EnvManagerType, os_environment::EnvironmentApi, python_environment::PythonEnvironmentKind, Locator, }; - use pet_python_utils::env::PythonEnv; use std::path::PathBuf; setup(); @@ -162,10 +161,9 @@ fn detect_new_conda_env() { fn detect_conda_env_from_path() { use pet_conda::Conda; use pet_core::{ - manager::EnvManagerType, os_environment::EnvironmentApi, + env::PythonEnv, manager::EnvManagerType, os_environment::EnvironmentApi, python_environment::PythonEnvironmentKind, Locator, }; - use pet_python_utils::env::PythonEnv; use std::path::PathBuf; setup(); diff --git a/crates/pet-conda/tests/lib_test.rs b/crates/pet-conda/tests/lib_test.rs index eb6524d4..7efa4ff1 100644 --- a/crates/pet-conda/tests/lib_test.rs +++ b/crates/pet-conda/tests/lib_test.rs @@ -8,8 +8,10 @@ mod common; fn find_conda_env_without_manager() { use common::{create_test_environment, resolve_test_path}; use pet_conda::Conda; - use pet_core::{self, arch::Architecture, python_environment::PythonEnvironmentKind, Locator}; - use pet_python_utils::env::PythonEnv; + use pet_core::{ + self, arch::Architecture, env::PythonEnv, python_environment::PythonEnvironmentKind, + Locator, + }; use std::collections::HashMap; let environment = create_test_environment(HashMap::new(), None, vec![], None); @@ -38,8 +40,10 @@ fn find_conda_env_without_manager() { fn find_conda_env_without_manager_but_detect_manager_from_history() { use common::{create_test_environment, resolve_test_path}; use pet_conda::Conda; - use pet_core::{self, arch::Architecture, python_environment::PythonEnvironmentKind, Locator}; - use pet_python_utils::env::PythonEnv; + use pet_core::{ + self, arch::Architecture, env::PythonEnv, python_environment::PythonEnvironmentKind, + Locator, + }; use std::{ collections::HashMap, fs::{self}, diff --git a/crates/pet-core/Cargo.toml b/crates/pet-core/Cargo.toml index 2f7d4b38..0e14864d 100644 --- a/crates/pet-core/Cargo.toml +++ b/crates/pet-core/Cargo.toml @@ -8,7 +8,6 @@ msvc_spectre_libs = { version = "0.1.1", features = ["error"] } [dependencies] pet-fs = { path = "../pet-fs" } -pet-python-utils = { path = "../pet-python-utils" } serde = { version = "1.0.152", features = ["derive"] } lazy_static = "1.4.0" regex = "1.10.4" diff --git a/crates/pet-core/src/env.rs b/crates/pet-core/src/env.rs new file mode 100644 index 00000000..f8ae69f6 --- /dev/null +++ b/crates/pet-core/src/env.rs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::path::PathBuf; + +use pet_fs::path::norm_case; + +use crate::pyvenv_cfg::PyVenvCfg; + +#[derive(Debug)] +pub struct PythonEnv { + /// Executable of the Python environment. + /// + /// Can be `/usr/bin/python` or `/opt/homebrew/bin/python3.12`. + /// Or even the fully (resolved &) qualified path to the python executable such as `/opt/homebrew/Cellar/python@3.12/3.12.3/Frameworks/Python.framework/Versions/3.12/bin/python3.12`. + /// + /// Note: This can be a symlink as well. + pub executable: PathBuf, + /// Environment prefix + pub prefix: Option, + /// Version of the Python environment. + pub version: Option, + /// Possible symlink (or known alternative link). + /// For instance: + /// + /// If `executable` is `/opt/homebrew/Cellar/python@3.12/3.12.3/Frameworks/Python.framework/Versions/3.12/bin/python3.12`, + /// then `symlink`` can be `/opt/homebrew/bin/python3.12` (or vice versa). + pub symlinks: Option>, +} + +impl PythonEnv { + pub fn new(executable: PathBuf, prefix: Option, version: Option) -> Self { + let mut prefix = prefix.clone(); + if let Some(value) = prefix { + prefix = norm_case(value).into(); + } + // if the prefix is not defined, try to get this. + // For instance, if the file is bin/python or Scripts/python + // And we have a pyvenv.cfg file in the parent directory, then we can get the prefix. + if prefix.is_none() { + let mut exe = executable.clone(); + exe.pop(); + if exe.ends_with("Scripts") || exe.ends_with("bin") { + exe.pop(); + if PyVenvCfg::find(&exe).is_some() { + prefix = Some(exe); + } + } + } + Self { + executable: norm_case(executable), + prefix, + version, + symlinks: None, + } + } +} diff --git a/crates/pet-core/src/lib.rs b/crates/pet-core/src/lib.rs index 436b4603..46fdfffc 100644 --- a/crates/pet-core/src/lib.rs +++ b/crates/pet-core/src/lib.rs @@ -3,15 +3,17 @@ use std::path::PathBuf; +use env::PythonEnv; use manager::EnvManager; -use pet_python_utils::env::PythonEnv; use python_environment::{PythonEnvironment, PythonEnvironmentKind}; use reporter::Reporter; pub mod arch; +pub mod env; pub mod manager; pub mod os_environment; pub mod python_environment; +pub mod pyvenv_cfg; pub mod reporter; pub mod telemetry; diff --git a/crates/pet-python-utils/src/pyvenv_cfg.rs b/crates/pet-core/src/pyvenv_cfg.rs similarity index 100% rename from crates/pet-python-utils/src/pyvenv_cfg.rs rename to crates/pet-core/src/pyvenv_cfg.rs diff --git a/crates/pet-homebrew/src/lib.rs b/crates/pet-homebrew/src/lib.rs index ce8093e5..1aedc0bf 100644 --- a/crates/pet-homebrew/src/lib.rs +++ b/crates/pet-homebrew/src/lib.rs @@ -5,13 +5,14 @@ use env_variables::EnvVariables; use environment_locations::get_homebrew_prefix_bin; use environments::get_python_info; use pet_core::{ + env::PythonEnv, os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, Locator, }; use pet_fs::path::resolve_symlink; -use pet_python_utils::{env::PythonEnv, executable::find_executables}; +use pet_python_utils::executable::find_executables; use pet_virtualenv::is_virtualenv; use std::{fs, path::PathBuf, thread}; use sym_links::is_homebrew_python; diff --git a/crates/pet-linux-global-python/src/lib.rs b/crates/pet-linux-global-python/src/lib.rs index 400f54c6..b42a691e 100644 --- a/crates/pet-linux-global-python/src/lib.rs +++ b/crates/pet-linux-global-python/src/lib.rs @@ -11,15 +11,13 @@ use std::{ use pet_core::{ arch::Architecture, + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, Locator, }; use pet_fs::path::resolve_symlink; -use pet_python_utils::{ - env::{PythonEnv, ResolvedPythonEnv}, - executable::find_executables, -}; +use pet_python_utils::{env::ResolvedPythonEnv, executable::find_executables}; use pet_virtualenv::is_virtualenv; pub struct LinuxGlobalPython { diff --git a/crates/pet-mac-commandlinetools/src/lib.rs b/crates/pet-mac-commandlinetools/src/lib.rs index 1676a3a3..0ad3aa63 100644 --- a/crates/pet-mac-commandlinetools/src/lib.rs +++ b/crates/pet-mac-commandlinetools/src/lib.rs @@ -2,16 +2,14 @@ // Licensed under the MIT License. use pet_core::{ + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, Locator, }; use pet_fs::path::resolve_symlink; use pet_python_utils::version; -use pet_python_utils::{ - env::{PythonEnv, ResolvedPythonEnv}, - executable::find_executables, -}; +use pet_python_utils::{env::ResolvedPythonEnv, executable::find_executables}; use pet_virtualenv::is_virtualenv; use std::path::PathBuf; diff --git a/crates/pet-mac-python-org/src/lib.rs b/crates/pet-mac-python-org/src/lib.rs index 76a64eb3..e9573d5c 100644 --- a/crates/pet-mac-python-org/src/lib.rs +++ b/crates/pet-mac-python-org/src/lib.rs @@ -2,12 +2,12 @@ // Licensed under the MIT License. use pet_core::{ + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, Locator, }; use pet_fs::path::resolve_symlink; -use pet_python_utils::env::PythonEnv; use pet_python_utils::executable::find_executables; use pet_python_utils::version; use pet_virtualenv::is_virtualenv; diff --git a/crates/pet-mac-xcode/src/lib.rs b/crates/pet-mac-xcode/src/lib.rs index 278a66cd..a6575dff 100644 --- a/crates/pet-mac-xcode/src/lib.rs +++ b/crates/pet-mac-xcode/src/lib.rs @@ -2,16 +2,14 @@ // Licensed under the MIT License. use pet_core::{ + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, Locator, }; use pet_fs::path::resolve_symlink; use pet_python_utils::version; -use pet_python_utils::{ - env::{PythonEnv, ResolvedPythonEnv}, - executable::find_executables, -}; +use pet_python_utils::{env::ResolvedPythonEnv, executable::find_executables}; use pet_virtualenv::is_virtualenv; use std::path::PathBuf; diff --git a/crates/pet-pipenv/src/lib.rs b/crates/pet-pipenv/src/lib.rs index 1738eefd..f64e2beb 100644 --- a/crates/pet-pipenv/src/lib.rs +++ b/crates/pet-pipenv/src/lib.rs @@ -2,6 +2,7 @@ // Licensed under the MIT License. use env_variables::EnvVariables; +use pet_core::env::PythonEnv; use pet_core::os_environment::Environment; use pet_core::{ python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, @@ -9,7 +10,6 @@ use pet_core::{ Locator, }; use pet_fs::path::norm_case; -use pet_python_utils::env::PythonEnv; use pet_python_utils::executable::find_executables; use pet_python_utils::version; use std::path::Path; diff --git a/crates/pet-poetry/src/lib.rs b/crates/pet-poetry/src/lib.rs index a1502d09..7610fe6e 100644 --- a/crates/pet-poetry/src/lib.rs +++ b/crates/pet-poetry/src/lib.rs @@ -5,12 +5,12 @@ use env_variables::EnvVariables; use environment_locations::list_environments; use manager::PoetryManager; use pet_core::{ + env::PythonEnv, os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, Configuration, Locator, LocatorResult, }; -use pet_python_utils::env::PythonEnv; use pet_virtualenv::is_virtualenv; use std::{ path::PathBuf, diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index de63de00..87fb5882 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -16,13 +16,14 @@ use environments::{get_generic_python_environment, get_virtual_env_environment}; use manager::PyEnvInfo; use pet_conda::{utils::is_conda_env, CondaLocator}; use pet_core::{ + env::PythonEnv, manager::{EnvManager, EnvManagerType}, os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, Locator, }; -use pet_python_utils::{env::PythonEnv, executable::find_executable}; +use pet_python_utils::executable::find_executable; pub mod env_variables; mod environment_locations; diff --git a/crates/pet-pyenv/tests/pyenv_test.rs b/crates/pet-pyenv/tests/pyenv_test.rs index 62ff87a0..ea46aee5 100644 --- a/crates/pet-pyenv/tests/pyenv_test.rs +++ b/crates/pet-pyenv/tests/pyenv_test.rs @@ -372,13 +372,13 @@ fn resolve_pyenv_environment() { use pet_conda::Conda; use pet_core::{ self, + env::PythonEnv, manager::{EnvManager, EnvManagerType}, python_environment::{PythonEnvironment, PythonEnvironmentKind}, Locator, }; use pet_pyenv; use pet_pyenv::PyEnv; - use pet_python_utils::env::PythonEnv; use std::{collections::HashMap, sync::Arc}; let home = resolve_test_path(&["unix", "pyenv", "user_home"]); diff --git a/crates/pet-python-utils/Cargo.toml b/crates/pet-python-utils/Cargo.toml index 6ec58b65..72046b29 100644 --- a/crates/pet-python-utils/Cargo.toml +++ b/crates/pet-python-utils/Cargo.toml @@ -10,6 +10,7 @@ msvc_spectre_libs = { version = "0.1.1", features = ["error"] } lazy_static = "1.4.0" regex = "1.10.4" pet-fs = { path = "../pet-fs" } +pet-core = { path = "../pet-core" } serde = { version = "1.0.152", features = ["derive"] } log = "0.4.21" serde_json = "1.0.93" diff --git a/crates/pet-python-utils/src/cache.rs b/crates/pet-python-utils/src/cache.rs new file mode 100644 index 00000000..1256ee9d --- /dev/null +++ b/crates/pet-python-utils/src/cache.rs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use lazy_static::lazy_static; +use pet_fs::path::norm_case; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, path::PathBuf, sync::{Arc, Mutex} +}; + +lazy_static! { + static ref CACHE: FSCache = FSCache::new(None); +} + +pub fn create_cache(executable: PathBuf) -> Arc>> { + CACHE.get(executable) +} + +pub fn get_cache_directory() -> Option { + CACHE.get_cache_directory() +} + +pub fn set_cache_directory(cache_dir: PathBuf) { + CACHE.set_cache_directory(cache_dir) +} + +pub fn generate_hash(executable: &PathBuf) -> String { + let mut hasher = Sha256::new(); + hasher.update(norm_case(executable).to_string_lossy().as_bytes()); + let h_bytes = hasher.finalize(); + // Convert 256 bits => Hext and then take 16 of the hex chars (that should be unique enough) + // We will handle collisions if they happen. + format!("{:x}", h_bytes)[..16].to_string() +} + +struct FSCache { + cache_dir: Arc>>, + locks: Mutex>>>>, +} + +impl FSCache { + pub fn new(cache_dir: Option) -> FSCache { + FSCache { + cache_dir: Arc::new(Mutex::new(cache_dir)), + locks: Mutex::new(HashMap::>>>::new()), + } + } + + pub fn get_cache_directory(&self) -> Option { + self.cache_dir.lock().unwrap().clone() + } + + /// Once a cache directory has been set, you cannot change it. + /// No point supporting such a scenario. + pub fn set_cache_directory(&self, cache_dir: PathBuf) { + self.cache_dir.lock().unwrap().replace(cache_dir); + } + pub fn create_cache(&self, executable: PathBuf) -> Arc>> { + match self.locks.lock().unwrap().entry(executable.clone()) { + Entry::Occupied(lock) => lock.get().clone(), + Entry::Vacant(lock) => { + let cache = Box::new(FSCacheEntry::new()) as Box<(dyn CacheEntry + 'static)>; + lock.insert(Arc::new(Mutex::new(cache))).clone() + } + } + } +} + +struct FSCacheEntry { + envoronment: Arc>>, +} +impl FSCacheEntry { + pub fn new() -> impl CacheEntry { + FSCacheEntry { + envoronment: Arc::new(Mutex::new(None)), + } + } +} +impl CacheEntry for FSCacheEntry { + fn get(&self) -> Option { + self.envoronment.lock().unwrap().clone() + } + + fn store(&self, environment: PythonEnvironment) { + self.envoronment.lock().unwrap().replace(environment); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[cfg(unix)] + fn test_hash_generation() { + assert_eq!( + generate_hash(&PathBuf::from( + "/Users/donjayamanne/demo/.venvTestInstall1/bin/python3.12" + )), + "e72c82125e7281e2" + ); + } + + #[test] + #[cfg(unix)] + fn test_hash_generation_upper_case() { + assert_eq!( + generate_hash(&PathBuf::from( + "/Users/donjayamanne/DEMO/.venvTestInstall1/bin/python3.12" + )), + "ecb0ee73d6ddfe97" + ); + } + + // #[test] + // #[cfg(unix)] + // fn test_hash_generation_upper_case() { + // let hashed_name = generate_env_name( + // "new-project", + // &"/Users/donjayamanne/temp/POETRY-UPPER/new-PROJECT".into(), + // ); + + // assert_eq!(hashed_name, "new-project-TbBV0MKD-py"); + // } + + // #[test] + // #[cfg(windows)] + // fn test_hash_generation_windows() { + // let hashed_name = generate_env_name( + // "demo-project1", + // &"C:\\temp\\poetry-folders\\demo-project1".into(), + // ); + + // assert_eq!(hashed_name, "demo-project1-f7sQRtG5-py"); + // } +} diff --git a/crates/pet-python-utils/src/env.rs b/crates/pet-python-utils/src/env.rs index 40797004..cf8b08c0 100644 --- a/crates/pet-python-utils/src/env.rs +++ b/crates/pet-python-utils/src/env.rs @@ -2,63 +2,14 @@ // Licensed under the MIT License. use log::{error, trace}; -use pet_fs::path::norm_case; +use pet_core::env::PythonEnv; use serde::Deserialize; use std::{ path::{Path, PathBuf}, time::SystemTime, }; -use crate::pyvenv_cfg::PyVenvCfg; - -#[derive(Debug)] -pub struct PythonEnv { - /// Executable of the Python environment. - /// - /// Can be `/usr/bin/python` or `/opt/homebrew/bin/python3.12`. - /// Or even the fully (resolved &) qualified path to the python executable such as `/opt/homebrew/Cellar/python@3.12/3.12.3/Frameworks/Python.framework/Versions/3.12/bin/python3.12`. - /// - /// Note: This can be a symlink as well. - pub executable: PathBuf, - /// Environment prefix - pub prefix: Option, - /// Version of the Python environment. - pub version: Option, - /// Possible symlink (or known alternative link). - /// For instance: - /// - /// If `executable` is `/opt/homebrew/Cellar/python@3.12/3.12.3/Frameworks/Python.framework/Versions/3.12/bin/python3.12`, - /// then `symlink`` can be `/opt/homebrew/bin/python3.12` (or vice versa). - pub symlinks: Option>, -} - -impl PythonEnv { - pub fn new(executable: PathBuf, prefix: Option, version: Option) -> Self { - let mut prefix = prefix.clone(); - if let Some(value) = prefix { - prefix = norm_case(value).into(); - } - // if the prefix is not defined, try to get this. - // For instance, if the file is bin/python or Scripts/python - // And we have a pyvenv.cfg file in the parent directory, then we can get the prefix. - if prefix.is_none() { - let mut exe = executable.clone(); - exe.pop(); - if exe.ends_with("Scripts") || exe.ends_with("bin") { - exe.pop(); - if PyVenvCfg::find(&exe).is_some() { - prefix = Some(exe); - } - } - } - Self { - executable: norm_case(executable), - prefix, - version, - symlinks: None, - } - } -} +use crate::cache::create_cache; const PYTHON_INFO_JSON_SEPARATOR: &str = "093385e9-59f7-4a16-a604-14bf206256fe"; const PYTHON_INFO_CMD:&str = "import json, sys; print('093385e9-59f7-4a16-a604-14bf206256fe');print(json.dumps({'version': '.'.join(str(n) for n in sys.version_info), 'sys_prefix': sys.prefix, 'executable': sys.executable, 'is64_bit': sys.maxsize > 2**32}))"; @@ -77,7 +28,7 @@ pub struct ResolvedPythonEnv { pub prefix: PathBuf, pub version: String, pub is64_bit: bool, - pub symlink: Option>, + pub symlinks: Option>, } impl ResolvedPythonEnv { @@ -87,61 +38,102 @@ impl ResolvedPythonEnv { Some(self.prefix.clone()), Some(self.version.clone()), ); - env.symlinks.clone_from(&self.symlink); + env.symlinks.clone_from(&self.symlinks); env } - pub fn from(executable: &Path) -> Option { - // Spawn the python exe and get the version, sys.prefix and sys.executable. - let executable = executable.to_str()?; - let start = SystemTime::now(); - trace!("Executing Python: {} -c {}", executable, PYTHON_INFO_CMD); - let result = std::process::Command::new(executable) - .args(["-c", PYTHON_INFO_CMD]) - .output(); - match result { - Ok(output) => { - let output = String::from_utf8(output.stdout).unwrap().trim().to_string(); - trace!( - "Executed Python {:?} in {:?} & produced an output {:?}", - executable, - start.elapsed(), - output - ); - if let Some((_, output)) = output.split_once(PYTHON_INFO_JSON_SEPARATOR) { - if let Ok(info) = serde_json::from_str::(output) { - Some(Self { - executable: PathBuf::from(info.executable.clone()), - prefix: PathBuf::from(info.sys_prefix), - version: info.version.trim().to_string(), - is64_bit: info.is64_bit, - symlink: if info.executable == executable { - None - } else { - Some(vec![PathBuf::from(executable)]) - }, - }) - } else { - error!( + /// Given the executable path, resolve the python environment by spawning python. + /// If we had previously spawned Python and we have the symlinks to this as well, + /// & all of them are the same as when this exe was previously spawned, + /// & mtime & ctimes of none of the exes (symlinks) have changed, then we can use the cached info. + pub fn from( + executable: &Path, + // known_symlinks: &Vec, + // cache: &dyn Cache, + ) -> Option { + let cache = create_cache(executable.to_path_buf()); + let entry = cache.lock().unwrap(); + if let Some(env) = entry.get() { + if let (Some(exe), Some(prefix), Some(version)) = + (env.executable, env.prefix, env.version) + { + // Ensure the given exe is in the list of symlinks. + if env + .symlinks + .clone() + .unwrap_or_default() + .contains(&executable.to_path_buf()) + { + return Some(ResolvedPythonEnv { + executable: exe, + is64_bit: false, + prefix, + symlinks: env.symlinks.clone(), + version, + }); + } + } + } + + if let Some(env) = get_interpreter_details(executable) { + entry.store(env); + Some(env) + } else { + None + } + } +} + +fn get_interpreter_details(executable: &Path) -> Option { + // Spawn the python exe and get the version, sys.prefix and sys.executable. + let executable = executable.to_str()?; + let start = SystemTime::now(); + trace!("Executing Python: {} -c {}", executable, PYTHON_INFO_CMD); + let result = std::process::Command::new(executable) + .args(["-c", PYTHON_INFO_CMD]) + .output(); + match result { + Ok(output) => { + let output = String::from_utf8(output.stdout).unwrap().trim().to_string(); + trace!( + "Executed Python {:?} in {:?} & produced an output {:?}", + executable, + start.elapsed(), + output + ); + if let Some((_, output)) = output.split_once(PYTHON_INFO_JSON_SEPARATOR) { + if let Ok(info) = serde_json::from_str::(output) { + Some(ResolvedPythonEnv { + executable: PathBuf::from(info.executable.clone()), + prefix: PathBuf::from(info.sys_prefix), + version: info.version.trim().to_string(), + is64_bit: info.is64_bit, + symlinks: if info.executable == executable { + None + } else { + Some(vec![PathBuf::from(executable)]) + }, + }) + } else { + error!( "Python Execution for {:?} produced an output {:?} that could not be parsed as JSON", executable, output, ); - None - } - } else { - error!( - "Python Execution for {:?} produced an output {:?} without a separator", - executable, output, - ); None } - } - Err(err) => { + } else { error!( - "Failed to execute Python to resolve info {:?}: {}", - executable, err + "Python Execution for {:?} produced an output {:?} without a separator", + executable, output, ); None } } + Err(err) => { + error!( + "Failed to execute Python to resolve info {:?}: {}", + executable, err + ); + None + } } } diff --git a/crates/pet-python-utils/src/lib.rs b/crates/pet-python-utils/src/lib.rs index 16872136..9816f24d 100644 --- a/crates/pet-python-utils/src/lib.rs +++ b/crates/pet-python-utils/src/lib.rs @@ -1,24 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +pub mod cache; pub mod env; pub mod executable; mod headers; pub mod platform_dirs; -pub mod pyvenv_cfg; pub mod version; - -pub fn add(left: usize, right: usize) -> usize { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} diff --git a/crates/pet-python-utils/src/version.rs b/crates/pet-python-utils/src/version.rs index c856faa8..b8b3ae4d 100644 --- a/crates/pet-python-utils/src/version.rs +++ b/crates/pet-python-utils/src/version.rs @@ -1,8 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::{headers::Headers, pyvenv_cfg::PyVenvCfg}; +use crate::headers::Headers; use log::{trace, warn}; +use pet_core::pyvenv_cfg::PyVenvCfg; use pet_fs::path::resolve_symlink; use std::{ path::{Path, PathBuf}, diff --git a/crates/pet-venv/src/lib.rs b/crates/pet-venv/src/lib.rs index 3ebe808b..a471914a 100644 --- a/crates/pet-venv/src/lib.rs +++ b/crates/pet-venv/src/lib.rs @@ -4,13 +4,14 @@ use std::path::Path; use pet_core::{ + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, + pyvenv_cfg::PyVenvCfg, reporter::Reporter, Locator, }; -use pet_python_utils::pyvenv_cfg::PyVenvCfg; +use pet_python_utils::executable::find_executables; use pet_python_utils::version; -use pet_python_utils::{env::PythonEnv, executable::find_executables}; fn is_venv_internal(env: &PythonEnv) -> Option { // env path cannot be empty. diff --git a/crates/pet-virtualenv/src/lib.rs b/crates/pet-virtualenv/src/lib.rs index a7adb0e3..c14dab1c 100644 --- a/crates/pet-virtualenv/src/lib.rs +++ b/crates/pet-virtualenv/src/lib.rs @@ -4,12 +4,13 @@ use std::path::Path; use pet_core::{ + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, Locator, }; +use pet_python_utils::executable::find_executables; use pet_python_utils::version; -use pet_python_utils::{env::PythonEnv, executable::find_executables}; pub fn is_virtualenv(env: &PythonEnv) -> bool { if env.prefix.is_none() { diff --git a/crates/pet-virtualenvwrapper/src/environments.rs b/crates/pet-virtualenvwrapper/src/environments.rs index 38bb73f6..d7f02aa7 100644 --- a/crates/pet-virtualenvwrapper/src/environments.rs +++ b/crates/pet-virtualenvwrapper/src/environments.rs @@ -2,8 +2,8 @@ // Licensed under the MIT License. use crate::{env_variables::EnvVariables, environment_locations::get_work_on_home_path}; +use pet_core::env::PythonEnv; use pet_fs::path::norm_case; -use pet_python_utils::env::PythonEnv; use pet_virtualenv::is_virtualenv; use std::{fs, path::PathBuf}; diff --git a/crates/pet-virtualenvwrapper/src/lib.rs b/crates/pet-virtualenvwrapper/src/lib.rs index d42fd22c..335c99ed 100644 --- a/crates/pet-virtualenvwrapper/src/lib.rs +++ b/crates/pet-virtualenvwrapper/src/lib.rs @@ -4,13 +4,14 @@ use env_variables::EnvVariables; use environments::{get_project, is_virtualenvwrapper}; use pet_core::{ + env::PythonEnv, os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, Locator, }; +use pet_python_utils::executable::find_executables; use pet_python_utils::version; -use pet_python_utils::{env::PythonEnv, executable::find_executables}; mod env_variables; mod environment_locations; diff --git a/crates/pet-windows-registry/src/lib.rs b/crates/pet-windows-registry/src/lib.rs index ee5e0269..9e8afcfc 100644 --- a/crates/pet-windows-registry/src/lib.rs +++ b/crates/pet-windows-registry/src/lib.rs @@ -5,11 +5,11 @@ use environments::get_registry_pythons; use pet_conda::{utils::is_conda_env, CondaLocator}; use pet_core::{ + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentKind}, reporter::Reporter, Locator, LocatorResult, }; -use pet_python_utils::env::PythonEnv; use pet_virtualenv::is_virtualenv; use std::sync::{Arc, Mutex}; diff --git a/crates/pet-windows-store/src/lib.rs b/crates/pet-windows-store/src/lib.rs index 5fdb5049..6e507961 100644 --- a/crates/pet-windows-store/src/lib.rs +++ b/crates/pet-windows-store/src/lib.rs @@ -8,10 +8,10 @@ mod environments; use crate::env_variables::EnvVariables; #[cfg(windows)] use environments::list_store_pythons; +use pet_core::env::PythonEnv; use pet_core::python_environment::{PythonEnvironment, PythonEnvironmentKind}; use pet_core::reporter::Reporter; use pet_core::{os_environment::Environment, Locator}; -use pet_python_utils::env::PythonEnv; use std::path::Path; use std::sync::{Arc, Mutex}; diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs index 58aeb8a6..457a5e18 100644 --- a/crates/pet/src/find.rs +++ b/crates/pet/src/find.rs @@ -3,12 +3,12 @@ use log::{trace, warn}; use pet_conda::utils::is_conda_env; +use pet_core::env::PythonEnv; use pet_core::os_environment::Environment; use pet_core::reporter::Reporter; use pet_core::{Configuration, Locator}; use pet_env_var_path::get_search_paths_from_env_variables; use pet_global_virtualenvs::list_global_virtual_envs_paths; -use pet_python_utils::env::PythonEnv; use pet_python_utils::executable::{ find_executable, find_executables, should_search_for_environments_in_path, }; diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index 814e915c..3e70c19f 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -3,8 +3,11 @@ use log::{error, info, trace}; use pet::resolve::resolve_environment; +use pet_cache::get_cache_directory; +use pet_cache::set_cache_directory; use pet_conda::Conda; use pet_conda::CondaLocator; +use pet_core::cache::Cache; use pet_core::python_environment::PythonEnvironment; use pet_core::telemetry::refresh_performance::RefreshPerformance; use pet_core::telemetry::TelemetryEvent; @@ -88,6 +91,8 @@ pub struct ConfigureOptions { /// Custom locations where environments can be found. Generally global locations where virtualenvs & the like can be found. /// Workspace directories should not be included into this list. pub environment_directories: Option>, + /// Directory to cache the Python environment details. + pub cache_directory: Option, } pub fn handle_configure(context: Arc, id: u32, params: Value) { @@ -100,6 +105,11 @@ pub fn handle_configure(context: Arc, id: u32, params: Value) { cfg.conda_executable = configure_options.conda_executable; cfg.environment_directories = configure_options.environment_directories; cfg.poetry_executable = configure_options.poetry_executable; + // We will not support changing the cache directories once set. + // No point, supporting such a use case. + if let Some(cache_directory) = configure_options.cache_directory { + set_cache_directory(cache_directory) + } trace!("Configuring locators: {:?}", cfg); drop(cfg); let config = context.configuration.read().unwrap().clone(); diff --git a/crates/pet/src/locators.rs b/crates/pet/src/locators.rs index 8b410847..fdb69031 100644 --- a/crates/pet/src/locators.rs +++ b/crates/pet/src/locators.rs @@ -4,6 +4,7 @@ use log::{info, trace}; use pet_conda::Conda; use pet_core::arch::Architecture; +use pet_core::env::PythonEnv; use pet_core::os_environment::Environment; use pet_core::python_environment::{ PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind, @@ -16,7 +17,7 @@ use pet_mac_xcode::MacXCode; use pet_pipenv::PipEnv; use pet_poetry::Poetry; use pet_pyenv::PyEnv; -use pet_python_utils::env::{PythonEnv, ResolvedPythonEnv}; +use pet_python_utils::env::ResolvedPythonEnv; use pet_venv::Venv; use pet_virtualenv::VirtualEnv; use pet_virtualenvwrapper::VirtualEnvWrapper; @@ -121,7 +122,7 @@ pub fn identify_python_environment_using_locators( // If one of the symlinks are in the PATH variable, then we can treat this as a GlobalPath kind. let symlinks = [ - resolved_env.symlink.clone().unwrap_or_default(), + resolved_env.symlinks.clone().unwrap_or_default(), vec![resolved_env.executable.clone(), executable.clone()], ] .concat(); diff --git a/crates/pet/src/resolve.rs b/crates/pet/src/resolve.rs index 93aefe4f..af716121 100644 --- a/crates/pet/src/resolve.rs +++ b/crates/pet/src/resolve.rs @@ -6,12 +6,13 @@ use std::{path::PathBuf, sync::Arc}; use log::{trace, warn}; use pet_core::{ arch::Architecture, + env::PythonEnv, os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentBuilder}, Locator, }; use pet_env_var_path::get_search_paths_from_env_variables; -use pet_python_utils::env::{PythonEnv, ResolvedPythonEnv}; +use pet_python_utils::env::ResolvedPythonEnv; use crate::locators::identify_python_environment_using_locators; @@ -45,7 +46,7 @@ pub fn resolve_environment( let discovered = env.clone(); let mut symlinks = env.symlinks.clone().unwrap_or_default(); symlinks.push(info.executable.clone()); - symlinks.append(&mut info.symlink.clone().unwrap_or_default()); + symlinks.append(&mut info.symlinks.clone().unwrap_or_default()); symlinks.sort(); symlinks.dedup(); diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index c4dee52f..8cf9ba4c 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -12,11 +12,11 @@ use pet::{ }; use pet_core::{ arch::Architecture, + env::PythonEnv, python_environment::{PythonEnvironment, PythonEnvironmentKind}, }; use pet_env_var_path::get_search_paths_from_env_variables; use pet_poetry::Poetry; -use pet_python_utils::env::PythonEnv; use pet_reporter::{ cache::{self, CacheReporter}, collect, From bf032d34cf4155e4c704bafc5c8ab2414955f195 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 15 Jul 2024 17:12:26 +1000 Subject: [PATCH 2/3] wip --- Cargo.lock | 1 + crates/pet-python-utils/Cargo.toml | 1 + crates/pet-python-utils/src/cache.rs | 37 ++++++++++++++++++++-------- crates/pet-python-utils/src/env.rs | 28 +++------------------ crates/pet/src/jsonrpc.rs | 4 +-- 5 files changed, 34 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b436fac..0a080d37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,6 +572,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", ] [[package]] diff --git a/crates/pet-python-utils/Cargo.toml b/crates/pet-python-utils/Cargo.toml index 72046b29..a782b5a6 100644 --- a/crates/pet-python-utils/Cargo.toml +++ b/crates/pet-python-utils/Cargo.toml @@ -14,3 +14,4 @@ pet-core = { path = "../pet-core" } serde = { version = "1.0.152", features = ["derive"] } log = "0.4.21" serde_json = "1.0.93" +sha2 = "0.10.6" diff --git a/crates/pet-python-utils/src/cache.rs b/crates/pet-python-utils/src/cache.rs index 1256ee9d..d4edafee 100644 --- a/crates/pet-python-utils/src/cache.rs +++ b/crates/pet-python-utils/src/cache.rs @@ -5,15 +5,25 @@ use lazy_static::lazy_static; use pet_fs::path::norm_case; use sha2::{Digest, Sha256}; use std::{ - collections::HashMap, path::PathBuf, sync::{Arc, Mutex} + collections::{hash_map::Entry, HashMap}, + path::PathBuf, + sync::{Arc, Mutex}, }; +use crate::env::ResolvedPythonEnv; + lazy_static! { static ref CACHE: FSCache = FSCache::new(None); } +pub trait CacheEntry: Send + Sync { + fn get(&self) -> Option; + fn store(&self, executable: PathBuf, environment: ResolvedPythonEnv); + fn track_symlinks(&self, executable: PathBuf, symlinks: Option>); +} + pub fn create_cache(executable: PathBuf) -> Arc>> { - CACHE.get(executable) + CACHE.create_cache(executable) } pub fn get_cache_directory() -> Option { @@ -33,16 +43,18 @@ pub fn generate_hash(executable: &PathBuf) -> String { format!("{:x}", h_bytes)[..16].to_string() } +pub type LockableCacheEntry = Arc>>; + struct FSCache { cache_dir: Arc>>, - locks: Mutex>>>>, + locks: Mutex>, } impl FSCache { pub fn new(cache_dir: Option) -> FSCache { FSCache { cache_dir: Arc::new(Mutex::new(cache_dir)), - locks: Mutex::new(HashMap::>>>::new()), + locks: Mutex::new(HashMap::::new()), } } @@ -55,11 +67,11 @@ impl FSCache { pub fn set_cache_directory(&self, cache_dir: PathBuf) { self.cache_dir.lock().unwrap().replace(cache_dir); } - pub fn create_cache(&self, executable: PathBuf) -> Arc>> { + pub fn create_cache(&self, executable: PathBuf) -> LockableCacheEntry { match self.locks.lock().unwrap().entry(executable.clone()) { Entry::Occupied(lock) => lock.get().clone(), Entry::Vacant(lock) => { - let cache = Box::new(FSCacheEntry::new()) as Box<(dyn CacheEntry + 'static)>; + let cache = Box::new(FSCacheEntry::create()) as Box<(dyn CacheEntry + 'static)>; lock.insert(Arc::new(Mutex::new(cache))).clone() } } @@ -67,23 +79,28 @@ impl FSCache { } struct FSCacheEntry { - envoronment: Arc>>, + envoronment: Arc>>, } impl FSCacheEntry { - pub fn new() -> impl CacheEntry { + pub fn create() -> impl CacheEntry { FSCacheEntry { envoronment: Arc::new(Mutex::new(None)), } } } + impl CacheEntry for FSCacheEntry { - fn get(&self) -> Option { + fn get(&self) -> Option { self.envoronment.lock().unwrap().clone() } - fn store(&self, environment: PythonEnvironment) { + fn store(&self, _executable: PathBuf, environment: ResolvedPythonEnv) { self.envoronment.lock().unwrap().replace(environment); } + + fn track_symlinks(&self, _executable: PathBuf, _symlinks: Option>) { + todo!() + } } #[cfg(test)] diff --git a/crates/pet-python-utils/src/env.rs b/crates/pet-python-utils/src/env.rs index cf8b08c0..94e10713 100644 --- a/crates/pet-python-utils/src/env.rs +++ b/crates/pet-python-utils/src/env.rs @@ -22,7 +22,7 @@ pub struct InterpreterInfo { pub is64_bit: bool, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ResolvedPythonEnv { pub executable: PathBuf, pub prefix: PathBuf, @@ -53,29 +53,9 @@ impl ResolvedPythonEnv { let cache = create_cache(executable.to_path_buf()); let entry = cache.lock().unwrap(); if let Some(env) = entry.get() { - if let (Some(exe), Some(prefix), Some(version)) = - (env.executable, env.prefix, env.version) - { - // Ensure the given exe is in the list of symlinks. - if env - .symlinks - .clone() - .unwrap_or_default() - .contains(&executable.to_path_buf()) - { - return Some(ResolvedPythonEnv { - executable: exe, - is64_bit: false, - prefix, - symlinks: env.symlinks.clone(), - version, - }); - } - } - } - - if let Some(env) = get_interpreter_details(executable) { - entry.store(env); + Some(env) + } else if let Some(env) = get_interpreter_details(executable) { + entry.store(executable.to_path_buf(), env.clone()); Some(env) } else { None diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index 3e70c19f..121076ad 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -3,11 +3,8 @@ use log::{error, info, trace}; use pet::resolve::resolve_environment; -use pet_cache::get_cache_directory; -use pet_cache::set_cache_directory; use pet_conda::Conda; use pet_conda::CondaLocator; -use pet_core::cache::Cache; use pet_core::python_environment::PythonEnvironment; use pet_core::telemetry::refresh_performance::RefreshPerformance; use pet_core::telemetry::TelemetryEvent; @@ -23,6 +20,7 @@ use pet_jsonrpc::{ }; use pet_poetry::Poetry; use pet_poetry::PoetryLocator; +use pet_python_utils::cache::set_cache_directory; use pet_reporter::collect; use pet_reporter::{cache::CacheReporter, jsonrpc}; use pet_telemetry::report_inaccuracies_identified_after_resolving; From ab948894cdfea6757baea95d637f71f9890292e9 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 15 Jul 2024 17:18:56 +1000 Subject: [PATCH 3/3] Fixes --- crates/pet-conda/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/pet-conda/src/lib.rs b/crates/pet-conda/src/lib.rs index ffc58ba9..00ac78f6 100644 --- a/crates/pet-conda/src/lib.rs +++ b/crates/pet-conda/src/lib.rs @@ -18,7 +18,6 @@ use pet_core::{ Locator, }; use pet_fs::path::norm_case; -use pet_python_utils::env::PythonEnv; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap,