Align cache directory calculation so that the LMDB store location is uniformly configurable

src/python/pants/base/build_environment.py

@@ -34,7 +34,9 @@ def get_buildroot() -> str:
 
 def get_pants_cachedir() -> str:
     """Return the pants global cache directory."""
-    # Follow the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html.
+    # TODO: Keep in alignment with rust `fs::default_cache_path`. This method
+    # is not used there directly because it would create a cycle for native bootstrap via
+    # BinaryUtil being used to download tools needed to bootstrap.
     cache_home = os.environ.get("XDG_CACHE_HOME")
     if not cache_home:
         cache_home = "~/.cache"
@@ -43,7 +45,9 @@ def get_pants_cachedir() -> str:
 
 def get_pants_configdir() -> str:
     """Return the pants global config directory."""
-    # Follow the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html.
+    # TODO: Keep in alignment with rust `fs::default_config_path`. This method
+    # is not used there directly because it would create a cycle for native bootstrap via
+    # BinaryUtil being used to download tools needed to bootstrap.
     config_home = os.environ.get("XDG_CONFIG_HOME")
     if not config_home:
         config_home = "~/.config"

src/python/pants/base/build_environment_test.py

@@ -0,0 +1,13 @@
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.base.build_environment import get_pants_cachedir, get_pants_configdir
+from pants.engine.internals.native import Native
+
+
+def test_get_pants_cachedir() -> None:
+    assert Native().default_cache_path() == get_pants_cachedir()
+
+
+def test_get_pants_configdir() -> None:
+    assert Native().default_config_path() == get_pants_configdir()

src/python/pants/engine/internals/native.py

@@ -127,6 +127,12 @@ def decompress_tarball(self, tarfile_path, dest_dir):
     def init_rust_logging(self, level, log_show_rust_3rdparty: bool, use_color: bool):
         return self.lib.init_logging(level, log_show_rust_3rdparty, use_color)
 
+    def default_cache_path(self) -> str:
+        return cast(str, self.lib.default_cache_path())
+
+    def default_config_path(self) -> str:
+        return cast(str, self.lib.default_config_path())
+
     def setup_pantsd_logger(self, log_file_path, level):
         return self.lib.setup_pantsd_logger(log_file_path, level)
 

src/python/pants/option/global_options.py

@@ -672,8 +672,9 @@ def register_bootstrap_options(cls, register):
             advanced=True,
             help="Directory to use for engine's local file store.",
             # This default is also hard-coded into the engine's rust code in
-            # fs::Store::default_path
-            default=os.path.expanduser("~/.cache/pants/lmdb_store"),
+            # fs::Store::default_path so that tools using a Store outside of pants
+            # are likely to be able to use the same storage location.
+            default=os.path.join(get_pants_cachedir(), "lmdb_store"),
         )
         register(
             "--local-execution-root-dir",

src/rust/engine/fs/Cargo.toml

@@ -8,6 +8,7 @@ publish = false
 [dependencies]
 async-trait = "0.1"
 bytes = "0.4.5"
+dirs = "1"
 futures = "0.3"
 glob = "0.2.11"
 ignore = "0.4.11"

src/rust/engine/fs/brfs/src/main.rs

@@ -667,11 +667,7 @@ pub fn mount<'a, P: AsRef<Path>>(
 async fn main() {
   env_logger::init();
 
-  let default_store_path = dirs::home_dir()
-    .expect("Couldn't find homedir")
-    .join(".cache")
-    .join("pants")
-    .join("lmdb_store");
+  let default_store_path = Store::default_path();
 
   let args = clap::App::new("brfs")
     .arg(

src/rust/engine/fs/src/lib.rs

@@ -60,6 +60,32 @@ lazy_static! {
 
 const TARGET_NOFILE_LIMIT: u64 = 10000;
 
+const XDG_CACHE_HOME: &str = "XDG_CACHE_HOME";
+const XDG_CONFIG_HOME: &str = "XDG_CONFIG_HOME";
+
+/// Follows the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html.
+pub fn default_cache_path() -> PathBuf {
+  // TODO: Keep in alignment with `pants.base.build_environment.get_pants_cachedir`. This method
+  // is not used there directly because of a cycle.
+  let cache_path = std::env::var(XDG_CACHE_HOME)
+    .ok()
+    .map(PathBuf::from)
+    .or_else(|| dirs::home_dir().map(|home| home.join(".cache")))
+    .unwrap_or_else(|| panic!("Could not find home dir or {}.", XDG_CACHE_HOME));
+  cache_path.join("pants")
+}
+
+/// Follows the unix XDB base spec: http://standards.freedesktop.org/basedir-spec/latest/index.html.
+pub fn default_config_path() -> PathBuf {
+  // TODO: Keep in alignment with `pants.base.build_environment.get_pants_configdir`.
+  let config_path = std::env::var(XDG_CONFIG_HOME)
+    .ok()
+    .map(PathBuf::from)
+    .or_else(|| dirs::home_dir().map(|home| home.join(".config")))
+    .unwrap_or_else(|| panic!("Could not find home dir or {}.", XDG_CONFIG_HOME));
+  config_path.join("pants")
+}
+
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub enum Stat {
   Link(Link),

src/rust/engine/fs/store/Cargo.toml

@@ -10,7 +10,6 @@ bazel_protos = { path = "../../process_execution/bazel_protos" }
 boxfuture = { path = "../../boxfuture" }
 bytes = "0.4.5"
 concrete_time = { path = "../../concrete_time" }
-dirs = "1"
 fs = { path = ".." }
 futures01 = { package = "futures", version = "0.1" }
 futures = { version = "0.3", features = ["compat"] }

src/rust/engine/fs/store/src/lib.rs

@@ -42,14 +42,15 @@ use bazel_protos::remote_execution as remexec;
 use boxfuture::{try_future, BoxFuture, Boxable};
 use bytes::Bytes;
 use concrete_time::TimeSpan;
-use fs::FileContent;
+use fs::{default_cache_path, FileContent};
 use futures::compat::Future01CompatExt;
 use futures::future::{self as future03, Either, FutureExt, TryFutureExt};
 use futures01::{future, Future};
 use hashing::Digest;
 use protobuf::Message;
 use serde_derive::Serialize;
 pub use serverset::BackoffConfig;
+
 use std::collections::{BTreeMap, HashMap};
 use std::convert::TryInto;
 use std::fs::OpenOptions;
@@ -272,12 +273,9 @@ impl Store {
     })
   }
 
-  // This default is also hard-coded into the Python options code in global_options.py
+  // This default suffix is also hard-coded into the Python options code in global_options.py
   pub fn default_path() -> PathBuf {
-    match dirs::home_dir() {
-      Some(home_dir) => home_dir.join(".cache").join("pants").join("lmdb_store"),
-      None => panic!("Could not find home dir"),
-    }
+    default_cache_path().join("lmdb_store")
   }
 
   ///

src/rust/engine/process_execution/src/named_caches.rs

@@ -1,6 +1,8 @@
 use std::collections::BTreeMap;
 use std::path::PathBuf;
 
+use fs::default_cache_path;
+
 use crate::RelativePath;
 
 #[derive(Clone, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)]
@@ -54,6 +56,11 @@ impl NamedCaches {
     NamedCaches { local_base }
   }
 
+  // This default suffix is also hard-coded into the Python options code in global_options.py
+  pub fn default_path() -> PathBuf {
+    default_cache_path().join("named_caches")
+  }
+
   ///
   /// Returns symlinks to create for the given set of NamedCaches.
   ///

src/rust/engine/process_executor/src/main.rs

@@ -28,18 +28,19 @@
 #![allow(clippy::mutex_atomic)]
 #![type_length_limit = "1076739"]
 
-use clap::{value_t, App, AppSettings, Arg};
-use futures::compat::Future01CompatExt;
-use hashing::{Digest, Fingerprint};
-use process_execution::{
-  Context, NamedCaches, Platform, PlatformConstraint, ProcessMetadata, RelativePath,
-};
 use std::collections::{BTreeMap, BTreeSet};
 use std::convert::TryFrom;
 use std::iter::{FromIterator, Iterator};
 use std::path::PathBuf;
 use std::process::exit;
 use std::time::Duration;
+
+use clap::{value_t, App, AppSettings, Arg};
+use futures::compat::Future01CompatExt;
+use hashing::{Digest, Fingerprint};
+use process_execution::{
+  Context, NamedCaches, Platform, PlatformConstraint, ProcessMetadata, RelativePath,
+};
 use store::{BackoffConfig, Store};
 use tokio::runtime::Handle;
 
@@ -269,8 +270,7 @@ async fn main() {
   let named_cache_path = args
     .value_of("named-cache-path")
     .map(PathBuf::from)
-    .or_else(|| dirs::home_dir().map(|home| home.join(".cache")))
-    .expect("Unable to locate a home directory, and no named-cache-path provided.");
+    .unwrap_or_else(NamedCaches::default_path);
   let server_arg = args.value_of("server");
   let remote_instance_arg = args.value_of("remote-instance-name").map(str::to_owned);
   let output_files = if let Some(values) = args.values_of("output-file-path") {

src/rust/engine/src/externs/interface.rs

@@ -80,6 +80,10 @@ py_module_initializer!(native_engine, |py, m| {
   m.add(py, "PollTimeout", py.get_type::<PollTimeout>())
     .unwrap();
 
+  m.add(py, "default_cache_path", py_fn!(py, default_cache_path()))?;
+
+  m.add(py, "default_config_path", py_fn!(py, default_config_path()))?;
+
   m.add(
     py,
     "init_logging",
@@ -1659,6 +1663,36 @@ fn materialize_directories(
   })
 }
 
+fn default_cache_path(py: Python) -> CPyResult<String> {
+  fs::default_cache_path()
+    .into_os_string()
+    .into_string()
+    .map_err(|s| {
+      PyErr::new::<exc::Exception, _>(
+        py,
+        (format!(
+          "Default cache path {:?} could not be converted to a string.",
+          s
+        ),),
+      )
+    })
+}
+
+fn default_config_path(py: Python) -> CPyResult<String> {
+  fs::default_config_path()
+    .into_os_string()
+    .into_string()
+    .map_err(|s| {
+      PyErr::new::<exc::Exception, _>(
+        py,
+        (format!(
+          "Default config path {:?} could not be converted to a string.",
+          s
+        ),),
+      )
+    })
+}
+
 fn init_logging(
   _: Python,
   level: u64,