Auto merge of #13926 - tweag:overlay, r=epage

Add local registry overlays

This PR adds (private to cargo internals) support for local registry overlays, in which you can locally pretend to add packages to remote registries; the local packages will have the same source ids as the remote registry that you're overlaying.

There are two ways to set up these overlays: programmatically using `GlobalContext::local_overlays` and through the `__CARGO_TEST_PACKAGE_CONFUSION_VULNERABILITY_DO_NOT_USE_THIS` environment variable. You can't set up these overlays with `.cargo/config`.

The motivation for this is [packaging workspaces](#10948). When we're packing a workspace, we'd like to be able to pretend (for lockfile generation and verification) that some workspace packages are already published even though they aren't.

crates/cargo-test-support/src/lib.rs

@@ -877,6 +877,17 @@ impl Execs {
         self
     }
 
+    pub fn overlay_registry(&mut self, url: &Url, path: &str) -> &mut Self {
+        if let Some(ref mut p) = self.process_builder {
+            let env_value = format!("{}={}", url, path);
+            p.env(
+                "__CARGO_TEST_DEPENDENCY_CONFUSION_VULNERABILITY_DO_NOT_USE_THIS",
+                env_value,
+            );
+        }
+        self
+    }
+
     pub fn enable_split_debuginfo_packed(&mut self) -> &mut Self {
         self.env("CARGO_PROFILE_DEV_SPLIT_DEBUGINFO", "packed")
             .env("CARGO_PROFILE_TEST_SPLIT_DEBUGINFO", "packed")

crates/cargo-test-support/src/registry.rs

@@ -1649,7 +1649,12 @@ impl Package {
     /// Returns the path to the compressed package file.
     pub fn archive_dst(&self) -> PathBuf {
         if self.local {
-            registry_path().join(format!("{}-{}.crate", self.name, self.vers))
+            let path = if self.alternative {
+                alt_registry_path()
+            } else {
+                registry_path()
+            };
+            path.join(format!("{}-{}.crate", self.name, self.vers))
         } else if self.alternative {
             alt_dl_path()
                 .join(&self.name)

crates/xtask-bump-check/src/xtask.rs

@@ -16,7 +16,6 @@ use std::fs;
 use std::task;
 
 use cargo::core::dependency::Dependency;
-use cargo::core::registry::PackageRegistry;
 use cargo::core::Package;
 use cargo::core::Registry;
 use cargo::core::SourceId;
@@ -137,7 +136,7 @@ fn bump_check(args: &clap::ArgMatches, gctx: &cargo::util::GlobalContext) -> Car
 
     let mut needs_bump = Vec::new();
 
-    check_crates_io(gctx, &changed_members, &mut needs_bump)?;
+    check_crates_io(&ws, &changed_members, &mut needs_bump)?;
 
     if let Some(referenced_commit) = referenced_commit.as_ref() {
         status(&format!("compare against `{}`", referenced_commit.id()))?;
@@ -385,12 +384,13 @@ fn symmetric_diff<'a>(
 ///
 /// Assumption: We always release a version larger than all existing versions.
 fn check_crates_io<'a>(
-    gctx: &GlobalContext,
+    ws: &Workspace<'a>,
     changed_members: &HashMap<&'a str, &'a Package>,
     needs_bump: &mut Vec<&'a Package>,
 ) -> CargoResult<()> {
+    let gctx = ws.gctx();
     let source_id = SourceId::crates_io(gctx)?;
-    let mut registry = PackageRegistry::new(gctx)?;
+    let mut registry = ws.package_registry()?;
     let _lock = gctx.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;
     registry.lock_patches();
     gctx.shell().status(

src/cargo/core/registry.rs

@@ -194,8 +194,10 @@ pub struct LockedPatchDependency {
 }
 
 impl<'gctx> PackageRegistry<'gctx> {
-    pub fn new(gctx: &'gctx GlobalContext) -> CargoResult<PackageRegistry<'gctx>> {
-        let source_config = SourceConfigMap::new(gctx)?;
+    pub fn new_with_source_config(
+        gctx: &'gctx GlobalContext,
+        source_config: SourceConfigMap<'gctx>,
+    ) -> CargoResult<PackageRegistry<'gctx>> {
         Ok(PackageRegistry {
             gctx,
             sources: SourceMap::new(),

src/cargo/core/workspace.rs

@@ -20,7 +20,7 @@ use crate::core::{
 };
 use crate::core::{EitherManifest, Package, SourceId, VirtualManifest};
 use crate::ops;
-use crate::sources::{PathSource, CRATES_IO_INDEX, CRATES_IO_REGISTRY};
+use crate::sources::{PathSource, SourceConfigMap, CRATES_IO_INDEX, CRATES_IO_REGISTRY};
 use crate::util::edit_distance;
 use crate::util::errors::{CargoResult, ManifestError};
 use crate::util::interning::InternedString;
@@ -109,6 +109,9 @@ pub struct Workspace<'gctx> {
 
     /// Workspace-level custom metadata
     custom_metadata: Option<toml::Value>,
+
+    /// Local overlay configuration. See [`crate::sources::overlay`].
+    local_overlays: HashMap<SourceId, PathBuf>,
 }
 
 // Separate structure for tracking loaded packages (to avoid loading anything
@@ -237,6 +240,7 @@ impl<'gctx> Workspace<'gctx> {
             resolve_behavior: ResolveBehavior::V1,
             resolve_honors_rust_version: false,
             custom_metadata: None,
+            local_overlays: HashMap::new(),
         }
     }
 
@@ -1674,6 +1678,44 @@ impl<'gctx> Workspace<'gctx> {
         // Cargo to panic, see issue #10545.
         self.is_member(&unit.pkg) && !(unit.target.for_host() || unit.pkg.proc_macro())
     }
+
+    /// Adds a local package registry overlaying a `SourceId`.
+    ///
+    /// See [`crate::sources::overlay::DependencyConfusionThreatOverlaySource`] for why you shouldn't use this.
+    pub fn add_local_overlay(&mut self, id: SourceId, registry_path: PathBuf) {
+        self.local_overlays.insert(id, registry_path);
+    }
+
+    /// Builds a package registry that reflects this workspace configuration.
+    pub fn package_registry(&self) -> CargoResult<PackageRegistry<'gctx>> {
+        let source_config =
+            SourceConfigMap::new_with_overlays(self.gctx(), self.local_overlays()?)?;
+        PackageRegistry::new_with_source_config(self.gctx(), source_config)
+    }
+
+    /// Returns all the configured local overlays, including the ones from our secret environment variable.
+    fn local_overlays(&self) -> CargoResult<impl Iterator<Item = (SourceId, SourceId)>> {
+        let mut ret = self
+            .local_overlays
+            .iter()
+            .map(|(id, path)| Ok((*id, SourceId::for_local_registry(path)?)))
+            .collect::<CargoResult<Vec<_>>>()?;
+
+        if let Ok(overlay) = self
+            .gctx
+            .get_env("__CARGO_TEST_DEPENDENCY_CONFUSION_VULNERABILITY_DO_NOT_USE_THIS")
+        {
+            let (url, path) = overlay.split_once('=').ok_or(anyhow::anyhow!(
+                "invalid overlay format. I won't tell you why; you shouldn't be using it anyway"
+            ))?;
+            ret.push((
+                SourceId::from_url(url)?,
+                SourceId::for_local_registry(path.as_ref())?,
+            ));
+        }
+
+        Ok(ret.into_iter())
+    }
 }
 
 impl<'gctx> Packages<'gctx> {

src/cargo/ops/cargo_add/mod.rs

@@ -78,7 +78,7 @@ pub fn add(workspace: &Workspace<'_>, options: &AddOptions<'_>) -> CargoResult<(
         );
     }
 
-    let mut registry = PackageRegistry::new(options.gctx)?;
+    let mut registry = workspace.package_registry()?;
 
     let deps = {
         let _lock = options

src/cargo/ops/cargo_package.rs

@@ -9,7 +9,7 @@ use std::task::Poll;
 use crate::core::compiler::{BuildConfig, CompileMode, DefaultExecutor, Executor};
 use crate::core::manifest::Target;
 use crate::core::resolver::CliFeatures;
-use crate::core::{registry::PackageRegistry, resolver::HasDevUnits};
+use crate::core::resolver::HasDevUnits;
 use crate::core::{Feature, PackageIdSpecQuery, Shell, Verbosity, Workspace};
 use crate::core::{Package, PackageId, PackageSet, Resolve, SourceId};
 use crate::sources::PathSource;
@@ -472,7 +472,7 @@ fn build_lock(ws: &Workspace<'_>, publish_pkg: &Package) -> CargoResult<String>
     let orig_resolve = ops::load_pkg_lockfile(ws)?;
 
     let tmp_ws = Workspace::ephemeral(publish_pkg.clone(), ws.gctx(), None, true)?;
-    let mut tmp_reg = PackageRegistry::new(ws.gctx())?;
+    let mut tmp_reg = ws.package_registry()?;
     let mut new_resolve = ops::resolve_with_previous(
         &mut tmp_reg,
         &tmp_ws,

src/cargo/ops/cargo_update.rs

@@ -32,7 +32,7 @@ pub struct UpdateOptions<'a> {
 }
 
 pub fn generate_lockfile(ws: &Workspace<'_>) -> CargoResult<()> {
-    let mut registry = PackageRegistry::new(ws.gctx())?;
+    let mut registry = ws.package_registry()?;
     let previous_resolve = None;
     let mut resolve = ops::resolve_with_previous(
         &mut registry,
@@ -73,7 +73,7 @@ pub fn update_lockfile(ws: &Workspace<'_>, opts: &UpdateOptions<'_>) -> CargoRes
                 // Precise option specified, so calculate a previous_resolve required
                 // by precise package update later.
                 Some(_) => {
-                    let mut registry = PackageRegistry::new(opts.gctx)?;
+                    let mut registry = ws.package_registry()?;
                     ops::resolve_with_previous(
                         &mut registry,
                         ws,
@@ -88,7 +88,7 @@ pub fn update_lockfile(ws: &Workspace<'_>, opts: &UpdateOptions<'_>) -> CargoRes
             }
         }
     };
-    let mut registry = PackageRegistry::new(opts.gctx)?;
+    let mut registry = ws.package_registry()?;
     let mut to_avoid = HashSet::new();
 
     if opts.to_update.is_empty() {
@@ -226,7 +226,7 @@ pub fn upgrade_manifests(
     // that we're synchronized against other Cargos.
     let _lock = gctx.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;
 
-    let mut registry = PackageRegistry::new(gctx)?;
+    let mut registry = ws.package_registry()?;
     registry.lock_patches();
 
     for member in ws.members_mut().sorted() {

src/cargo/ops/resolve.rs

@@ -116,7 +116,7 @@ version. This may also occur with an optional dependency that is not enabled.";
 /// This is a simple interface used by commands like `clean`, `fetch`, and
 /// `package`, which don't specify any options or features.
 pub fn resolve_ws<'a>(ws: &Workspace<'a>, dry_run: bool) -> CargoResult<(PackageSet<'a>, Resolve)> {
-    let mut registry = PackageRegistry::new(ws.gctx())?;
+    let mut registry = ws.package_registry()?;
     let resolve = resolve_with_registry(ws, &mut registry, dry_run)?;
     let packages = get_resolved_packages(&resolve, registry)?;
     Ok((packages, resolve))
@@ -142,7 +142,7 @@ pub fn resolve_ws_with_opts<'gctx>(
     force_all_targets: ForceAllTargets,
     dry_run: bool,
 ) -> CargoResult<WorkspaceResolve<'gctx>> {
-    let mut registry = PackageRegistry::new(ws.gctx())?;
+    let mut registry = ws.package_registry()?;
     let (resolve, resolved_with_overrides) = if ws.ignore_lock() {
         let add_patches = true;
         let resolve = None;

src/cargo/sources/config.rs

@@ -5,6 +5,7 @@
 //! sources to one another via the `replace-with` key in `.cargo/config`.
 
 use crate::core::{GitReference, PackageId, SourceId};
+use crate::sources::overlay::DependencyConfusionThreatOverlaySource;
 use crate::sources::source::Source;
 use crate::sources::{ReplacedSource, CRATES_IO_REGISTRY};
 use crate::util::context::{self, ConfigRelativePath, OptValue};
@@ -24,6 +25,8 @@ pub struct SourceConfigMap<'gctx> {
     cfgs: HashMap<String, SourceConfig>,
     /// Mapping of [`SourceId`] to the source name.
     id2name: HashMap<SourceId, String>,
+    /// Mapping of sources to local registries that will be overlaid on them.
+    overlays: HashMap<SourceId, SourceId>,
     gctx: &'gctx GlobalContext,
 }
 
@@ -81,6 +84,18 @@ impl<'gctx> SourceConfigMap<'gctx> {
                 base.add_config(key, value)?;
             }
         }
+
+        Ok(base)
+    }
+
+    /// Like [`SourceConfigMap::new`] but includes sources from source
+    /// replacement configurations.
+    pub fn new_with_overlays(
+        gctx: &'gctx GlobalContext,
+        overlays: impl IntoIterator<Item = (SourceId, SourceId)>,
+    ) -> CargoResult<SourceConfigMap<'gctx>> {
+        let mut base = SourceConfigMap::new(gctx)?;
+        base.overlays = overlays.into_iter().collect();
         Ok(base)
     }
 
@@ -90,6 +105,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
         let mut base = SourceConfigMap {
             cfgs: HashMap::new(),
             id2name: HashMap::new(),
+            overlays: HashMap::new(),
             gctx,
         };
         base.add(
@@ -136,7 +152,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
         debug!("loading: {}", id);
 
         let Some(mut name) = self.id2name.get(&id) else {
-            return id.load(self.gctx, yanked_whitelist);
+            return self.load_overlaid(id, yanked_whitelist);
         };
         let mut cfg_loc = "";
         let orig_name = name;
@@ -161,7 +177,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
                     name = s;
                     cfg_loc = c;
                 }
-                None if id == cfg.id => return id.load(self.gctx, yanked_whitelist),
+                None if id == cfg.id => return self.load_overlaid(id, yanked_whitelist),
                 None => {
                     break cfg.id.with_precise_from(id);
                 }
@@ -178,8 +194,8 @@ impl<'gctx> SourceConfigMap<'gctx> {
             }
         };
 
-        let new_src = new_id.load(
-            self.gctx,
+        let new_src = self.load_overlaid(
+            new_id,
             &yanked_whitelist
                 .iter()
                 .map(|p| p.map_source(id, new_id))
@@ -215,6 +231,23 @@ restore the source replacement configuration to continue the build
         Ok(Box::new(ReplacedSource::new(id, new_id, new_src)))
     }
 
+    /// Gets the [`Source`] for a given [`SourceId`] without performing any source replacement.
+    fn load_overlaid(
+        &self,
+        id: SourceId,
+        yanked_whitelist: &HashSet<PackageId>,
+    ) -> CargoResult<Box<dyn Source + 'gctx>> {
+        let src = id.load(self.gctx, yanked_whitelist)?;
+        if let Some(overlay_id) = self.overlays.get(&id) {
+            let overlay = overlay_id.load(self.gctx(), yanked_whitelist)?;
+            Ok(Box::new(DependencyConfusionThreatOverlaySource::new(
+                overlay, src,
+            )))
+        } else {
+            Ok(src)
+        }
+    }
+
     /// Adds a source config with an associated name.
     fn add(&mut self, name: &str, cfg: SourceConfig) -> CargoResult<()> {
         if let Some(old_name) = self.id2name.insert(cfg.id, name.to_string()) {

src/cargo/sources/mod.rs

@@ -39,6 +39,7 @@ pub use self::replaced::ReplacedSource;
 pub mod config;
 pub mod directory;
 pub mod git;
+pub mod overlay;
 pub mod path;
 pub mod registry;
 pub mod replaced;