diff --git a/src/python/pants/base/build_environment.py b/src/python/pants/base/build_environment.py index 0e755611947..d93a3c404b1 100644 --- a/src/python/pants/base/build_environment.py +++ b/src/python/pants/base/build_environment.py @@ -34,7 +34,9 @@ def get_buildroot() -> str: def get_pants_cachedir() -> str: """Return the pants global cache directory.""" - # Follow the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html. + # TODO: Keep in alignment with rust `fs::default_cache_path`. This method + # is not used there directly because it would create a cycle for native bootstrap via + # BinaryUtil being used to download tools needed to bootstrap. cache_home = os.environ.get("XDG_CACHE_HOME") if not cache_home: cache_home = "~/.cache" @@ -43,7 +45,9 @@ def get_pants_cachedir() -> str: def get_pants_configdir() -> str: """Return the pants global config directory.""" - # Follow the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html. + # TODO: Keep in alignment with rust `fs::default_config_path`. This method + # is not used there directly because it would create a cycle for native bootstrap via + # BinaryUtil being used to download tools needed to bootstrap. config_home = os.environ.get("XDG_CONFIG_HOME") if not config_home: config_home = "~/.config" diff --git a/src/python/pants/base/build_environment_test.py b/src/python/pants/base/build_environment_test.py new file mode 100644 index 00000000000..073baf233a6 --- /dev/null +++ b/src/python/pants/base/build_environment_test.py @@ -0,0 +1,13 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pants.base.build_environment import get_pants_cachedir, get_pants_configdir +from pants.engine.internals.native import Native + + +def test_get_pants_cachedir() -> None: + assert Native().default_cache_path() == get_pants_cachedir() + + +def test_get_pants_configdir() -> None: + assert Native().default_config_path() == get_pants_configdir() diff --git a/src/python/pants/engine/internals/native.py b/src/python/pants/engine/internals/native.py index aa871b479ca..7bea0ee38ad 100644 --- a/src/python/pants/engine/internals/native.py +++ b/src/python/pants/engine/internals/native.py @@ -127,6 +127,12 @@ def decompress_tarball(self, tarfile_path, dest_dir): def init_rust_logging(self, level, log_show_rust_3rdparty: bool, use_color: bool): return self.lib.init_logging(level, log_show_rust_3rdparty, use_color) + def default_cache_path(self) -> str: + return cast(str, self.lib.default_cache_path()) + + def default_config_path(self) -> str: + return cast(str, self.lib.default_config_path()) + def setup_pantsd_logger(self, log_file_path, level): return self.lib.setup_pantsd_logger(log_file_path, level) diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 7324f4e5a67..713f516db53 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -672,8 +672,9 @@ def register_bootstrap_options(cls, register): advanced=True, help="Directory to use for engine's local file store.", # This default is also hard-coded into the engine's rust code in - # fs::Store::default_path - default=os.path.expanduser("~/.cache/pants/lmdb_store"), + # fs::Store::default_path so that tools using a Store outside of pants + # are likely to be able to use the same storage location. + default=os.path.join(get_pants_cachedir(), "lmdb_store"), ) register( "--local-execution-root-dir", diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock index fbe23d23718..953786ed5a9 100644 --- a/src/rust/engine/Cargo.lock +++ b/src/rust/engine/Cargo.lock @@ -789,6 +789,7 @@ version = "0.0.1" dependencies = [ "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "ignore 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2719,7 +2720,6 @@ dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "concrete_time 0.0.1", "criterion 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "fs 0.0.1", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/src/rust/engine/fs/Cargo.toml b/src/rust/engine/fs/Cargo.toml index f5ef56d0343..8b255190eb1 100644 --- a/src/rust/engine/fs/Cargo.toml +++ b/src/rust/engine/fs/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] async-trait = "0.1" bytes = "0.4.5" +dirs = "1" futures = "0.3" glob = "0.2.11" ignore = "0.4.11" diff --git a/src/rust/engine/fs/brfs/src/main.rs b/src/rust/engine/fs/brfs/src/main.rs index f855e3924a7..be04fe98975 100644 --- a/src/rust/engine/fs/brfs/src/main.rs +++ b/src/rust/engine/fs/brfs/src/main.rs @@ -667,11 +667,7 @@ pub fn mount<'a, P: AsRef>( async fn main() { env_logger::init(); - let default_store_path = dirs::home_dir() - .expect("Couldn't find homedir") - .join(".cache") - .join("pants") - .join("lmdb_store"); + let default_store_path = Store::default_path(); let args = clap::App::new("brfs") .arg( diff --git a/src/rust/engine/fs/src/lib.rs b/src/rust/engine/fs/src/lib.rs index 6d01e244c00..5b1de81337c 100644 --- a/src/rust/engine/fs/src/lib.rs +++ b/src/rust/engine/fs/src/lib.rs @@ -60,6 +60,32 @@ lazy_static! { const TARGET_NOFILE_LIMIT: u64 = 10000; +const XDG_CACHE_HOME: &str = "XDG_CACHE_HOME"; +const XDG_CONFIG_HOME: &str = "XDG_CONFIG_HOME"; + +/// Follows the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html. +pub fn default_cache_path() -> PathBuf { + // TODO: Keep in alignment with `pants.base.build_environment.get_pants_cachedir`. This method + // is not used there directly because of a cycle. + let cache_path = std::env::var(XDG_CACHE_HOME) + .ok() + .map(PathBuf::from) + .or_else(|| dirs::home_dir().map(|home| home.join(".cache"))) + .unwrap_or_else(|| panic!("Could not find home dir or {}.", XDG_CACHE_HOME)); + cache_path.join("pants") +} + +/// Follows the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html. +pub fn default_config_path() -> PathBuf { + // TODO: Keep in alignment with `pants.base.build_environment.get_pants_configdir`. + let config_path = std::env::var(XDG_CONFIG_HOME) + .ok() + .map(PathBuf::from) + .or_else(|| dirs::home_dir().map(|home| home.join(".config"))) + .unwrap_or_else(|| panic!("Could not find home dir or {}.", XDG_CONFIG_HOME)); + config_path.join("pants") +} + #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub enum Stat { Link(Link), diff --git a/src/rust/engine/fs/store/Cargo.toml b/src/rust/engine/fs/store/Cargo.toml index 712024bdefa..678bd2492c1 100644 --- a/src/rust/engine/fs/store/Cargo.toml +++ b/src/rust/engine/fs/store/Cargo.toml @@ -10,7 +10,6 @@ bazel_protos = { path = "../../process_execution/bazel_protos" } boxfuture = { path = "../../boxfuture" } bytes = "0.4.5" concrete_time = { path = "../../concrete_time" } -dirs = "1" fs = { path = ".." } futures01 = { package = "futures", version = "0.1" } futures = { version = "0.3", features = ["compat"] } diff --git a/src/rust/engine/fs/store/src/lib.rs b/src/rust/engine/fs/store/src/lib.rs index 21bc882ea02..71fc732ac0a 100644 --- a/src/rust/engine/fs/store/src/lib.rs +++ b/src/rust/engine/fs/store/src/lib.rs @@ -42,7 +42,7 @@ use bazel_protos::remote_execution as remexec; use boxfuture::{try_future, BoxFuture, Boxable}; use bytes::Bytes; use concrete_time::TimeSpan; -use fs::FileContent; +use fs::{default_cache_path, FileContent}; use futures::compat::Future01CompatExt; use futures::future::{self as future03, Either, FutureExt, TryFutureExt}; use futures01::{future, Future}; @@ -50,6 +50,7 @@ use hashing::Digest; use protobuf::Message; use serde_derive::Serialize; pub use serverset::BackoffConfig; + use std::collections::{BTreeMap, HashMap}; use std::convert::TryInto; use std::fs::OpenOptions; @@ -272,12 +273,9 @@ impl Store { }) } - // This default is also hard-coded into the Python options code in global_options.py + // This default suffix is also hard-coded into the Python options code in global_options.py pub fn default_path() -> PathBuf { - match dirs::home_dir() { - Some(home_dir) => home_dir.join(".cache").join("pants").join("lmdb_store"), - None => panic!("Could not find home dir"), - } + default_cache_path().join("lmdb_store") } /// diff --git a/src/rust/engine/process_execution/src/named_caches.rs b/src/rust/engine/process_execution/src/named_caches.rs index c3c4c124d18..f2bafd8a569 100644 --- a/src/rust/engine/process_execution/src/named_caches.rs +++ b/src/rust/engine/process_execution/src/named_caches.rs @@ -1,6 +1,8 @@ use std::collections::BTreeMap; use std::path::PathBuf; +use fs::default_cache_path; + use crate::RelativePath; #[derive(Clone, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)] @@ -54,6 +56,11 @@ impl NamedCaches { NamedCaches { local_base } } + // This default suffix is also hard-coded into the Python options code in global_options.py + pub fn default_path() -> PathBuf { + default_cache_path().join("named_caches") + } + /// /// Returns symlinks to create for the given set of NamedCaches. /// diff --git a/src/rust/engine/process_executor/src/main.rs b/src/rust/engine/process_executor/src/main.rs index f818608cefa..a11381ae8f1 100644 --- a/src/rust/engine/process_executor/src/main.rs +++ b/src/rust/engine/process_executor/src/main.rs @@ -28,18 +28,19 @@ #![allow(clippy::mutex_atomic)] #![type_length_limit = "1076739"] -use clap::{value_t, App, AppSettings, Arg}; -use futures::compat::Future01CompatExt; -use hashing::{Digest, Fingerprint}; -use process_execution::{ - Context, NamedCaches, Platform, PlatformConstraint, ProcessMetadata, RelativePath, -}; use std::collections::{BTreeMap, BTreeSet}; use std::convert::TryFrom; use std::iter::{FromIterator, Iterator}; use std::path::PathBuf; use std::process::exit; use std::time::Duration; + +use clap::{value_t, App, AppSettings, Arg}; +use futures::compat::Future01CompatExt; +use hashing::{Digest, Fingerprint}; +use process_execution::{ + Context, NamedCaches, Platform, PlatformConstraint, ProcessMetadata, RelativePath, +}; use store::{BackoffConfig, Store}; use tokio::runtime::Handle; @@ -269,8 +270,7 @@ async fn main() { let named_cache_path = args .value_of("named-cache-path") .map(PathBuf::from) - .or_else(|| dirs::home_dir().map(|home| home.join(".cache"))) - .expect("Unable to locate a home directory, and no named-cache-path provided."); + .unwrap_or_else(NamedCaches::default_path); let server_arg = args.value_of("server"); let remote_instance_arg = args.value_of("remote-instance-name").map(str::to_owned); let output_files = if let Some(values) = args.values_of("output-file-path") { diff --git a/src/rust/engine/src/externs/interface.rs b/src/rust/engine/src/externs/interface.rs index b7592ba68f4..368474df7b7 100644 --- a/src/rust/engine/src/externs/interface.rs +++ b/src/rust/engine/src/externs/interface.rs @@ -80,6 +80,10 @@ py_module_initializer!(native_engine, |py, m| { m.add(py, "PollTimeout", py.get_type::()) .unwrap(); + m.add(py, "default_cache_path", py_fn!(py, default_cache_path()))?; + + m.add(py, "default_config_path", py_fn!(py, default_config_path()))?; + m.add( py, "init_logging", @@ -1659,6 +1663,36 @@ fn materialize_directories( }) } +fn default_cache_path(py: Python) -> CPyResult { + fs::default_cache_path() + .into_os_string() + .into_string() + .map_err(|s| { + PyErr::new::( + py, + (format!( + "Default cache path {:?} could not be converted to a string.", + s + ),), + ) + }) +} + +fn default_config_path(py: Python) -> CPyResult { + fs::default_config_path() + .into_os_string() + .into_string() + .map_err(|s| { + PyErr::new::( + py, + (format!( + "Default config path {:?} could not be converted to a string.", + s + ),), + ) + }) +} + fn init_logging( _: Python, level: u64,