Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(index): Don't talk to crates.io when using alt registry #735

Merged
merged 2 commits into from
Dec 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/ops/cargo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,18 +112,25 @@ pub fn publish(

pub fn wait_for_publish(
index: &mut crate::ops::index::CratesIoIndex,
registry: Option<&str>,
name: &str,
version: &str,
timeout: std::time::Duration,
dry_run: bool,
) -> CargoResult<()> {
if !dry_run {
if registry.is_none() {
// HACK: `index` never reports crates as present for alternative registries
log::debug!("Not waiting for publish as that is only supported for crates.io; ensure you are using at least cargo v1.66 which will wait for you.");
return Ok(());
}

let now = std::time::Instant::now();
let sleep_time = std::time::Duration::from_secs(1);
let mut logged = false;
loop {
index.update_krate(name);
if is_published(index, name, version) {
index.update_krate(registry, name);
if is_published(index, registry, name, version) {
break;
} else if timeout < now.elapsed() {
anyhow::bail!("timeout waiting for crate to be published");
Expand All @@ -145,10 +152,11 @@ pub fn wait_for_publish(

pub fn is_published(
index: &mut crate::ops::index::CratesIoIndex,
registry: Option<&str>,
name: &str,
version: &str,
) -> bool {
match index.has_krate_version(name, version) {
match index.has_krate_version(registry, name, version) {
Ok(has_krate_version) => has_krate_version.unwrap_or(false),
Err(err) => {
// For both http and git indices, this _might_ be an error that goes away in
Expand Down
43 changes: 32 additions & 11 deletions src/ops/index.rs
Original file line number Diff line number Diff line change
@@ -1,53 +1,74 @@
use tame_index::krate::IndexKrate;
use tame_index::utils::flock::FileLock;

#[derive(Default)]
pub struct CratesIoIndex {
index: RemoteIndex,
index: Option<RemoteIndex>,
cache: std::collections::HashMap<String, Option<IndexKrate>>,
}

impl CratesIoIndex {
#[inline]
pub fn open() -> Result<Self, crate::error::CliError> {
Ok(Self {
index: RemoteIndex::open()?,
cache: Default::default(),
})
pub fn new() -> Self {
Self {
index: None,
cache: std::collections::HashMap::new(),
}
}

/// Determines if the specified crate exists in the crates.io index
#[inline]
pub fn has_krate(&mut self, name: &str) -> Result<bool, crate::error::CliError> {
Ok(self.krate(name)?.map(|_| true).unwrap_or(false))
pub fn has_krate(
&mut self,
registry: Option<&str>,
name: &str,
) -> Result<bool, crate::error::CliError> {
Ok(self.krate(registry, name)?.map(|_| true).unwrap_or(false))
}

/// Determines if the specified crate version exists in the crates.io index
#[inline]
pub fn has_krate_version(
&mut self,
registry: Option<&str>,
name: &str,
version: &str,
) -> Result<Option<bool>, crate::error::CliError> {
let krate = self.krate(name)?;
let krate = self.krate(registry, name)?;
Ok(krate.map(|ik| ik.versions.iter().any(|iv| iv.version == version)))
}

#[inline]
pub fn update_krate(&mut self, name: &str) {
pub fn update_krate(&mut self, registry: Option<&str>, name: &str) {
if registry.is_some() {
return;
}

self.cache.remove(name);
}

pub(crate) fn krate(
&mut self,
registry: Option<&str>,
name: &str,
) -> Result<Option<IndexKrate>, crate::error::CliError> {
if let Some(registry) = registry {
log::trace!("Cannot connect to registry `{registry}`");
return Ok(None);
}

if let Some(entry) = self.cache.get(name) {
log::trace!("Reusing index for {name}");
return Ok(entry.clone());
}

if self.index.is_none() {
log::trace!("Connecting to index");
self.index = Some(RemoteIndex::open()?);
}
let index = self.index.as_mut().unwrap();
log::trace!("Downloading index for {name}");
let entry = self.index.krate(name)?;
let entry = index.krate(name)?;
self.cache.insert(name.to_owned(), entry.clone());
Ok(entry)
}
Expand Down
3 changes: 2 additions & 1 deletion src/steps/hook.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ pub struct HookStep {
impl HookStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let mut index = crate::ops::index::CratesIoIndex::open()?;
let mut index = crate::ops::index::CratesIoIndex::new();

if self.dry_run {
let _ =
Expand Down Expand Up @@ -88,6 +88,7 @@ impl HookStep {
let version = &pkg.initial_version;
if !crate::ops::cargo::is_published(
&mut index,
pkg.config.registry(),
crate_name,
&version.full_version_string,
) {
Expand Down
3 changes: 2 additions & 1 deletion src/steps/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,9 +225,10 @@ pub fn verify_rate_limit(
let mut new = 0;
let mut existing = 0;
for pkg in pkgs {
// Note: these rate limits are only known for default registry
if pkg.config.registry().is_none() && pkg.config.publish() {
let crate_name = pkg.meta.name.as_str();
if index.has_krate(crate_name)? {
if index.has_krate(None, crate_name)? {
existing += 1;
} else {
new += 1;
Expand Down
55 changes: 27 additions & 28 deletions src/steps/publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,14 @@ impl PublishStep {

let mut pkgs = plan::plan(pkgs)?;

let mut index = crate::ops::index::CratesIoIndex::open()?;
let mut index = crate::ops::index::CratesIoIndex::new();
for pkg in pkgs.values_mut() {
if pkg.config.registry().is_none() && pkg.config.release() {
if pkg.config.release() {
let crate_name = pkg.meta.name.as_str();
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
if crate::ops::cargo::is_published(
&mut index,
pkg.config.registry(),
crate_name,
&version.full_version_string,
) {
Expand Down Expand Up @@ -200,33 +201,31 @@ pub fn publish(
return Err(101.into());
}

if pkg.config.registry().is_none() {
let timeout = std::time::Duration::from_secs(300);
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
crate::ops::cargo::wait_for_publish(
index,
crate_name,
&version.full_version_string,
timeout,
dry_run,
)?;
// HACK: Even once the index is updated, there seems to be another step before the publish is fully ready.
// We don't have a way yet to check for that, so waiting for now in hopes everything is ready
if !dry_run {
let publish_grace_sleep = std::env::var("PUBLISH_GRACE_SLEEP")
.unwrap_or_else(|_| Default::default())
.parse()
.unwrap_or(0);
if 0 < publish_grace_sleep {
log::debug!(
"waiting an additional {} seconds for crates.io to update its indices...",
publish_grace_sleep
);
std::thread::sleep(std::time::Duration::from_secs(publish_grace_sleep));
}
let timeout = std::time::Duration::from_secs(300);
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
crate::ops::cargo::wait_for_publish(
index,
pkg.config.registry(),
crate_name,
&version.full_version_string,
timeout,
dry_run,
)?;
// HACK: Even once the index is updated, there seems to be another step before the publish is fully ready.
// We don't have a way yet to check for that, so waiting for now in hopes everything is ready
if !dry_run {
let publish_grace_sleep = std::env::var("PUBLISH_GRACE_SLEEP")
.unwrap_or_else(|_| Default::default())
.parse()
.unwrap_or(0);
if 0 < publish_grace_sleep {
log::debug!(
"waiting an additional {} seconds for {} to update its indices...",
publish_grace_sleep,
pkg.config.registry().unwrap_or("crates.io")
);
std::thread::sleep(std::time::Duration::from_secs(publish_grace_sleep));
}
} else {
log::debug!("not waiting for publish because the registry is not crates.io and doesn't get updated automatically");
}
}

Expand Down
42 changes: 28 additions & 14 deletions src/steps/release.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ pub struct ReleaseStep {
impl ReleaseStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let mut index = crate::ops::index::CratesIoIndex::open()?;
let mut index = crate::ops::index::CratesIoIndex::new();

if self.dry_run {
let _ =
Expand All @@ -73,7 +73,7 @@ impl ReleaseStep {
pkg.bump(level_or_version, self.metadata.as_deref())?;
}
}
if index.has_krate(&pkg.meta.name)? {
if index.has_krate(pkg.config.registry(), &pkg.meta.name)? {
// Already published, skip it. Use `cargo release owner` for one-time updates
pkg.ensure_owners = false;
}
Expand Down Expand Up @@ -101,7 +101,12 @@ impl ReleaseStep {
&& !explicitly_excluded
{
let version = &pkg.initial_version;
if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
if !cargo::is_published(
&mut index,
pkg.config.registry(),
crate_name,
&version.full_version_string,
) {
log::debug!(
"enabled {}, v{} is unpublished",
crate_name,
Expand Down Expand Up @@ -152,10 +157,16 @@ impl ReleaseStep {
continue;
};

// HACK: `index` only supports default registry
if pkg.config.publish() && pkg.config.registry().is_none() {
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
let crate_name = pkg.meta.name.as_str();
if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
if !cargo::is_published(
&mut index,
pkg.config.registry(),
crate_name,
&version.full_version_string,
) {
let _ = crate::ops::shell::warn(format!(
"disabled by user, skipping {} v{} despite being unpublished",
crate_name, version.full_version_string,
Expand Down Expand Up @@ -195,16 +206,19 @@ impl ReleaseStep {
if !pkg.config.publish() {
continue;
}
if pkg.config.registry().is_none() {
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
let crate_name = pkg.meta.name.as_str();
if cargo::is_published(&mut index, crate_name, &version.full_version_string) {
let _ = crate::ops::shell::error(format!(
"{} {} is already published",
crate_name, version.full_version_string
));
double_publish = true;
}
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
let crate_name = pkg.meta.name.as_str();
if cargo::is_published(
&mut index,
pkg.config.registry(),
crate_name,
&version.full_version_string,
) {
let _ = crate::ops::shell::error(format!(
"{} {} is already published",
crate_name, version.full_version_string
));
double_publish = true;
}
}
if double_publish {
Expand Down
3 changes: 2 additions & 1 deletion src/steps/replace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ pub struct ReplaceStep {
impl ReplaceStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let mut index = crate::ops::index::CratesIoIndex::open()?;
let mut index = crate::ops::index::CratesIoIndex::new();

if self.dry_run {
let _ =
Expand Down Expand Up @@ -84,6 +84,7 @@ impl ReplaceStep {
let version = &pkg.initial_version;
if !crate::ops::cargo::is_published(
&mut index,
pkg.config.registry(),
crate_name,
&version.full_version_string,
) {
Expand Down
Loading