diff --git a/crates/crates_io_smoke_test/src/cargo.rs b/crates/crates_io_smoke_test/src/cargo.rs
index 539406d0cea..d2e5797587e 100644
--- a/crates/crates_io_smoke_test/src/cargo.rs
+++ b/crates/crates_io_smoke_test/src/cargo.rs
@@ -1,6 +1,7 @@
 use crate::exit_status_ext::ExitStatusExt;
 use secrecy::{ExposeSecret, SecretString};
 use std::path::Path;
+use std::process::Output;
 use tokio::process::Command;
 
 #[allow(unstable_name_collisions)]
@@ -43,3 +44,21 @@ pub async fn publish(project_path: &Path, token: &SecretString) -> anyhow::Resul
         .exit_ok()
         .map_err(Into::into)
 }
+
+pub async fn publish_with_output(
+    project_path: &Path,
+    token: &SecretString,
+) -> anyhow::Result<Output> {
+    Command::new("cargo")
+        .args(["publish", "--registry", "staging"])
+        .current_dir(project_path)
+        .env("CARGO_TERM_COLOR", "always")
+        .env(
+            "CARGO_REGISTRIES_STAGING_INDEX",
+            "https://github.com/rust-lang/staging.crates.io-index",
+        )
+        .env("CARGO_REGISTRIES_STAGING_TOKEN", token.expose_secret())
+        .output()
+        .await
+        .map_err(Into::into)
+}
diff --git a/crates/crates_io_smoke_test/src/main.rs b/crates/crates_io_smoke_test/src/main.rs
index e3b3d7005b0..a469f5fb82b 100644
--- a/crates/crates_io_smoke_test/src/main.rs
+++ b/crates/crates_io_smoke_test/src/main.rs
@@ -7,7 +7,7 @@ mod git;
 extern crate tracing;
 
 use crate::api::ApiClient;
-use anyhow::{anyhow, Context};
+use anyhow::{anyhow, bail, Context};
 use clap::Parser;
 use secrecy::SecretString;
 use std::path::{Path, PathBuf};
@@ -65,6 +65,20 @@ async fn main() -> anyhow::Result<()> {
         .await
         .context("Failed to create project")?;
 
+    info!("Checking publish with invalid authentication…");
+    let invalid_token = "invalid-token".into();
+    let output = cargo::publish_with_output(&project_path, &invalid_token).await?;
+    if output.status.success() {
+        bail!("Expected `cargo publish` to fail with invalid token");
+    } else {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        if !stderr.contains("401 Unauthorized")
+            || !stderr.contains("The given API token does not match the format used by crates.io")
+        {
+            bail!("Expected `cargo publish` to fail with an `401 Unauthorized` error, but got: {stderr}");
+        }
+    }
+
     if options.skip_publish {
         info!("Packaging crate file…");
         cargo::package(&project_path)
@@ -72,6 +86,7 @@ async fn main() -> anyhow::Result<()> {
             .context("Failed to run `cargo package`")?;
 
         info!("Skipping publish step");
+        new_version = old_version;
     } else {
         info!("Publishing to staging.crates.io…");
         cargo::publish(&project_path, &options.token)