Auto merge of #12247 - weihanglo:source-doc, r=epage

docs: doc comments for all registry kinds

src/cargo/sources/registry/download.rs

@@ -1,3 +1,8 @@
+//! Shared download logic between [`HttpRegistry`] and [`RemoteRegistry`].
+//!
+//! [`HttpRegistry`]: super::http_remote::HttpRegistry
+//! [`RemoteRegistry`]: super::remote::RemoteRegistry
+
 use anyhow::Context;
 use cargo_util::Sha256;
 
@@ -20,10 +25,15 @@ const PREFIX_TEMPLATE: &str = "{prefix}";
 const LOWER_PREFIX_TEMPLATE: &str = "{lowerprefix}";
 const CHECKSUM_TEMPLATE: &str = "{sha256-checksum}";
 
+/// Filename of the `.crate` tarball, e.g., `once_cell-1.18.0.crate`.
 pub(super) fn filename(pkg: PackageId) -> String {
     format!("{}-{}.crate", pkg.name(), pkg.version())
 }
 
+/// Checks if `pkg` is downloaded and ready under the directory at `cache_path`.
+/// If not, returns a URL to download it from.
+///
+/// This is primarily called by [`RegistryData::download`](super::RegistryData::download).
 pub(super) fn download(
     cache_path: &Filesystem,
     config: &Config,
@@ -86,6 +96,10 @@ pub(super) fn download(
     })
 }
 
+/// Verifies the integrity of `data` with `checksum` and persists it under the
+/// directory at `cache_path`.
+///
+/// This is primarily called by [`RegistryData::finish_download`](super::RegistryData::finish_download).
 pub(super) fn finish_download(
     cache_path: &Filesystem,
     config: &Config,
@@ -119,6 +133,10 @@ pub(super) fn finish_download(
     Ok(dst)
 }
 
+/// Checks if a tarball of `pkg` has been already downloaded under the
+/// directory at `cache_path`.
+///
+/// This is primarily called by [`RegistryData::is_crate_downloaded`](super::RegistryData::is_crate_downloaded).
 pub(super) fn is_crate_downloaded(
     cache_path: &Filesystem,
     config: &Config,

src/cargo/sources/registry/http_remote.rs

@@ -1,6 +1,4 @@
-//! Access to a HTTP-based crate registry.
-//!
-//! See [`HttpRegistry`] for details.
+//! Access to a HTTP-based crate registry. See [`HttpRegistry`] for details.
 
 use crate::core::{PackageId, SourceId};
 use crate::ops::{self};
@@ -52,8 +50,15 @@ const UNKNOWN: &'static str = "Unknown";
 ///
 /// [RFC 2789]: https://github.com/rust-lang/rfcs/pull/2789
 pub struct HttpRegistry<'cfg> {
+    /// Path to the registry index (`$CARGO_HOME/registry/index/$REG-HASH`).
+    ///
+    /// To be fair, `HttpRegistry` doesn't store the registry index it
+    /// downloads on the file system, but other cached data like registry
+    /// configuration could be stored here.
     index_path: Filesystem,
+    /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/cache/$REG-HASH`).
     cache_path: Filesystem,
+    /// The unique identifier of this registry source.
     source_id: SourceId,
     config: &'cfg Config,
 
@@ -95,20 +100,20 @@ pub struct HttpRegistry<'cfg> {
     quiet: bool,
 }
 
-/// Helper for downloading crates.
+/// State for currently pending index file downloads.
 struct Downloads<'cfg> {
     /// When a download is started, it is added to this map. The key is a
-    /// "token" (see `Download::token`). It is removed once the download is
+    /// "token" (see [`Download::token`]). It is removed once the download is
     /// finished.
     pending: HashMap<usize, (Download<'cfg>, EasyHandle)>,
     /// Set of paths currently being downloaded.
-    /// This should stay in sync with `pending`.
+    /// This should stay in sync with the `pending` field.
     pending_paths: HashSet<PathBuf>,
     /// Downloads that have failed and are waiting to retry again later.
     sleeping: SleepTracker<(Download<'cfg>, Easy)>,
     /// The final result of each download.
     results: HashMap<PathBuf, CargoResult<CompletedDownload>>,
-    /// The next ID to use for creating a token (see `Download::token`).
+    /// The next ID to use for creating a token (see [`Download::token`]).
     next: usize,
     /// Progress bar.
     progress: RefCell<Option<Progress<'cfg>>>,
@@ -119,9 +124,10 @@ struct Downloads<'cfg> {
     blocking_calls: usize,
 }
 
+/// Represents a single index file download, including its progress and retry.
 struct Download<'cfg> {
-    /// The token for this download, used as the key of the `Downloads::pending` map
-    /// and stored in `EasyHandle` as well.
+    /// The token for this download, used as the key of the
+    /// [`Downloads::pending`] map and stored in [`EasyHandle`] as well.
     token: usize,
 
     /// The path of the package that we're downloading.
@@ -137,28 +143,39 @@ struct Download<'cfg> {
     retry: Retry<'cfg>,
 }
 
+/// HTTPS headers [`HttpRegistry`] cares about.
 #[derive(Default)]
 struct Headers {
     last_modified: Option<String>,
     etag: Option<String>,
     www_authenticate: Vec<String>,
+    /// We don't care about these headers. Put them here for debugging purpose.
     others: Vec<String>,
 }
 
+/// HTTP status code [`HttpRegistry`] cares about.
 enum StatusCode {
     Success,
     NotModified,
     NotFound,
     Unauthorized,
 }
 
+/// Represents a complete [`Download`] from an HTTP request.
+///
+/// Usually it is constructed in [`HttpRegistry::handle_completed_downloads`],
+/// and then returns to the caller of [`HttpRegistry::load()`].
 struct CompletedDownload {
     response_code: StatusCode,
     data: Vec<u8>,
     header_map: Headers,
 }
 
 impl<'cfg> HttpRegistry<'cfg> {
+    /// Creates a HTTP-rebased remote registry for `source_id`.
+    ///
+    /// * `name` --- Name of a path segment where `.crate` tarballs and the
+    ///   registry index are stored. Expect to be unique.
     pub fn new(
         source_id: SourceId,
         config: &'cfg Config,
@@ -208,6 +225,7 @@ impl<'cfg> HttpRegistry<'cfg> {
         })
     }
 
+    /// Splits HTTP `HEADER: VALUE` to a tuple.
     fn handle_http_header(buf: &[u8]) -> Option<(&str, &str)> {
         if buf.is_empty() {
             return None;
@@ -222,6 +240,9 @@ impl<'cfg> HttpRegistry<'cfg> {
         Some((tag, value))
     }
 
+    /// Setup the necessary works before the first fetch gets started.
+    ///
+    /// This is a no-op if called more than one time.
     fn start_fetch(&mut self) -> CargoResult<()> {
         if self.fetch_started {
             // We only need to run the setup code once.
@@ -249,6 +270,8 @@ impl<'cfg> HttpRegistry<'cfg> {
         Ok(())
     }
 
+    /// Checks the results inside the [`HttpRegistry::multi`] handle, and
+    /// updates relevant state in [`HttpRegistry::downloads`] accordingly.
     fn handle_completed_downloads(&mut self) -> CargoResult<()> {
         assert_eq!(
             self.downloads.pending.len(),
@@ -322,11 +345,15 @@ impl<'cfg> HttpRegistry<'cfg> {
         Ok(())
     }
 
+    /// Constructs the full URL to download a index file.
     fn full_url(&self, path: &Path) -> String {
         // self.url always ends with a slash.
         format!("{}{}", self.url, path.display())
     }
 
+    /// Check if an index file of `path` is up-to-date.
+    ///
+    /// The `path` argument is the same as in [`RegistryData::load`].
     fn is_fresh(&self, path: &Path) -> bool {
         if !self.requested_update {
             trace!(
@@ -373,7 +400,7 @@ impl<'cfg> HttpRegistry<'cfg> {
         Ok(self.registry_config.as_ref())
     }
 
-    /// Get the registry configuration.
+    /// Get the registry configuration from either cache or remote.
     fn config(&mut self) -> Poll<CargoResult<&RegistryConfig>> {
         debug!("loading config");
         let index_path = self.assert_index_locked(&self.index_path);
@@ -405,6 +432,7 @@ impl<'cfg> HttpRegistry<'cfg> {
         }
     }
 
+    /// Moves failed [`Download`]s that are ready to retry to the pending queue.
     fn add_sleepers(&mut self) -> CargoResult<()> {
         for (dl, handle) in self.downloads.sleeping.to_retry() {
             let mut handle = self.multi.add(handle)?;
@@ -790,6 +818,7 @@ impl<'cfg> RegistryData for HttpRegistry<'cfg> {
 }
 
 impl<'cfg> Downloads<'cfg> {
+    /// Updates the state of the progress bar for downloads.
     fn tick(&self) -> CargoResult<()> {
         let mut progress = self.progress.borrow_mut();
         let Some(progress) = progress.as_mut() else { return Ok(()); };

src/cargo/sources/registry/local.rs

@@ -1,3 +1,5 @@
+//! Access to a regstiry on the local filesystem. See [`LocalRegistry`] for more.
+
 use crate::core::PackageId;
 use crate::sources::registry::{LoadResponse, MaybeLock, RegistryConfig, RegistryData};
 use crate::util::errors::CargoResult;
@@ -10,18 +12,68 @@ use std::path::Path;
 use std::task::Poll;
 
 /// A local registry is a registry that lives on the filesystem as a set of
-/// `.crate` files with an `index` directory in the same format as a remote
+/// `.crate` files with an `index` directory in the [same format] as a remote
 /// registry.
+///
+/// This type is primarily accessed through the [`RegistryData`] trait.
+///
+/// When a local registry is requested for a package, it simply looks into what
+/// its index has under the `index` directory. When [`LocalRegistry::download`]
+/// is called, a local registry verifies the checksum of the requested `.crate`
+/// tarball and then unpacks it to `$CARGO_HOME/.registry/src`.
+///
+/// > Note that there is a third-party subcommand [`cargo-local-registry`],
+/// > which happened to be developed by a former Cargo team member when local
+/// > registry was introduced. The tool is to ease the burden of maintaining
+/// > local registries. However, in general the Cargo team avoids recommending
+/// > any specific third-party crate. Just FYI.
+///
+/// [same format]: super#the-format-of-the-index
+/// [`cargo-local-registry`]: https://crates.io/crates/cargo-local-registry
+///
+/// # Filesystem hierarchy
+///
+/// Here is an example layout of a local registry on a local filesystem:
+///
+/// ```text
+/// [registry root]/
+/// ├── index/                      # registry index
+/// │  ├── an/
+/// │  │  └── yh/
+/// │  │     └── anyhow
+/// │  ├── ru/
+/// │  │  └── st/
+/// │  │     ├── rustls
+/// │  │     └── rustls-ffi
+/// │  └── se/
+/// │     └── mv/
+/// │        └── semver
+/// ├── anyhow-1.0.71.crate         # pre-downloaded crate tarballs
+/// ├── rustls-0.20.8.crate
+/// ├── rustls-ffi-0.8.2.crate
+/// └── semver-1.0.17.crate
+/// ```
+///
+/// For general concepts of registries, see the [module-level documentation](crate::sources::registry).
 pub struct LocalRegistry<'cfg> {
+    /// Path to the registry index.
     index_path: Filesystem,
+    /// Root path of this local registry.
     root: Filesystem,
+    /// Path where this local registry extract `.crate` tarballs to.
     src_path: Filesystem,
     config: &'cfg Config,
+    /// Whether this source has updated all package informations it may contain.
     updated: bool,
+    /// Disables status messages.
     quiet: bool,
 }
 
 impl<'cfg> LocalRegistry<'cfg> {
+    /// Creates a local registry at `root`.
+    ///
+    /// * `name` --- Name of a path segment where `.crate` tarballs are stored.
+    ///   Expect to be unique.
     pub fn new(root: &Path, config: &'cfg Config, name: &str) -> LocalRegistry<'cfg> {
         LocalRegistry {
             src_path: config.registry_source_path().join(name),

src/cargo/sources/registry/mod.rs

@@ -277,6 +277,9 @@ pub struct RegistryConfig {
     /// will be extended with `/{crate}/{version}/download` to
     /// support registries like crates.io which were created before the
     /// templating setup was created.
+    ///
+    /// For more on the template of the download URL, see [Index Configuration](
+    /// https://doc.rust-lang.org/nightly/cargo/reference/registry-index.html#index-configuration).
     pub dl: String,
 
     /// API endpoint for the registry. This is what's actually hit to perform
@@ -485,8 +488,10 @@ impl<'cfg> RegistrySource<'cfg> {
 
     /// Creates a source of a registry. This is a inner helper function.
     ///
-    /// * `name` --- Unique name for this source to store source files (`.crate` tarballs) are stored.
+    /// * `name` --- Name of a path segment which may affect where `.crate`
+    ///   tarballs, the registry index and cache are stored. Expect to be unique.
     /// * `ops` --- The underlying [`RegistryData`] type.
+    /// * `yanked_whitelist` --- Packages allowed to be used, even if they are yanked.
     fn new(
         source_id: SourceId,
         config: &'cfg Config,