Merge pull request #12328 from Turbo87/fastly-crate

Extract `crates_io_fastly` workspace crate

Author: Turbo87

Cargo.lock

@@ -73,6 +73,7 @@ crates_io_database_dump = { path = "crates/crates_io_database_dump" }
 crates_io_diesel_helpers = { path = "crates/crates_io_diesel_helpers" }
 crates_io_docs_rs = { path = "crates/crates_io_docs_rs" }
 crates_io_env_vars = { path = "crates/crates_io_env_vars" }
+crates_io_fastly = { path = "crates/crates_io_fastly" }
 crates_io_github = { path = "crates/crates_io_github" }
 crates_io_heroku = { path = "crates/crates_io_heroku" }
 crates_io_index = { path = "crates/crates_io_index" }

Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "crates_io_fastly"
+version = "0.0.0"
+license = "MIT OR Apache-2.0"
+edition = "2024"
+
+[lints]
+workspace = true
+
+[dependencies]
+reqwest = { version = "=0.12.24", features = ["json"] }
+secrecy = "=0.10.3"
+thiserror = "=2.0.17"
+tracing = "=0.1.41"

crates/crates_io_fastly/Cargo.toml

@@ -0,0 +1,13 @@
+# crates_io_fastly
+
+This package implements functionality for interacting with the Fastly API.
+
+The `Fastly` struct provides methods for purging cached content on Fastly's CDN.
+It uses the `reqwest` crate to perform HTTP requests to the Fastly API and
+authenticates using an API token.
+
+The main operations supported are:
+- `purge()` - Purge a specific path on a single domain
+- `purge_both_domains()` - Purge a path on both the primary and prefixed domains
+
+Note that wildcard invalidations are not supported by the Fastly API.

crates/crates_io_fastly/README.md

@@ -0,0 +1,124 @@
+#![doc = include_str!("../README.md")]
+
+use reqwest::Client;
+use reqwest::header::{HeaderValue, InvalidHeaderValue};
+use secrecy::{ExposeSecret, SecretString};
+use thiserror::Error;
+use tracing::{debug, instrument, trace};
+
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("Wildcard invalidations are not supported for Fastly")]
+    WildcardNotSupported,
+
+    #[error("Invalid API token format")]
+    InvalidApiToken(#[from] InvalidHeaderValue),
+
+    #[error("Failed to `POST {url}`{}: {source}", status.map(|s| format!(" (status: {})", s)).unwrap_or_default())]
+    PurgeFailed {
+        url: String,
+        status: Option<reqwest::StatusCode>,
+        #[source]
+        source: reqwest::Error,
+    },
+}
+
+#[derive(Debug)]
+pub struct Fastly {
+    client: Client,
+    api_token: SecretString,
+}
+
+impl Fastly {
+    pub fn new(api_token: SecretString) -> Self {
+        let client = Client::new();
+        Self { client, api_token }
+    }
+
+    /// Invalidate a path on Fastly
+    ///
+    /// This method takes a path and invalidates the cached content on Fastly. The path must not
+    /// contain a wildcard, since the Fastly API does not support wildcard invalidations. Paths are
+    /// invalidated for both domains that are associated with the Fastly service.
+    ///
+    /// Requests are authenticated using a token that is sent in a header. The token is passed to
+    /// the application as an environment variable.
+    ///
+    /// More information on Fastly's APIs for cache invalidations can be found here:
+    /// <https://developer.fastly.com/reference/api/purging/>
+    #[instrument(skip(self))]
+    pub async fn purge_both_domains(&self, base_domain: &str, path: &str) -> Result<(), Error> {
+        self.purge(base_domain, path).await?;
+
+        let prefixed_domain = format!("fastly-{base_domain}");
+        self.purge(&prefixed_domain, path).await?;
+
+        Ok(())
+    }
+
+    /// Invalidate a path on Fastly
+    ///
+    /// This method takes a domain and path and invalidates the cached content
+    /// on Fastly. The path must not contain a wildcard, since the Fastly API
+    /// does not support wildcard invalidations.
+    ///
+    /// More information on Fastly's APIs for cache invalidations can be found here:
+    /// <https://developer.fastly.com/reference/api/purging/>
+    #[instrument(skip(self))]
+    pub async fn purge(&self, domain: &str, path: &str) -> Result<(), Error> {
+        if path.contains('*') {
+            return Err(Error::WildcardNotSupported);
+        }
+
+        let path = path.trim_start_matches('/');
+        let url = format!("https://api.fastly.com/purge/{domain}/{path}");
+
+        trace!(?url);
+
+        debug!("sending invalidation request to Fastly");
+        let response = self
+            .client
+            .post(&url)
+            .header("Fastly-Key", self.token_header_value()?)
+            .send()
+            .await
+            .map_err(|source| Error::PurgeFailed {
+                url: url.clone(),
+                status: None,
+                source,
+            })?;
+
+        let status = response.status();
+
+        match response.error_for_status_ref() {
+            Ok(_) => {
+                debug!(?status, "invalidation request accepted by Fastly");
+                Ok(())
+            }
+            Err(error) => {
+                let headers = response.headers().clone();
+                let body = response.text().await;
+                debug!(
+                    ?status,
+                    ?headers,
+                    ?body,
+                    "invalidation request to Fastly failed"
+                );
+
+                Err(Error::PurgeFailed {
+                    url,
+                    status: Some(status),
+                    source: error,
+                })
+            }
+        }
+    }
+
+    fn token_header_value(&self) -> Result<HeaderValue, InvalidHeaderValue> {
+        let api_token = self.api_token.expose_secret();
+
+        let mut header_value = HeaderValue::try_from(api_token)?;
+        header_value.set_sensitive(true);
+        Ok(header_value)
+    }
+}

crates/crates_io_fastly/src/lib.rs

@@ -16,19 +16,18 @@ extern crate tracing;
 use anyhow::Context;
 use crates_io::app::create_database_pool;
 use crates_io::cloudfront::CloudFront;
-use crates_io::fastly::Fastly;
 use crates_io::ssh;
 use crates_io::storage::Storage;
 use crates_io::worker::{Environment, RunnerExt};
 use crates_io::{Emails, config};
 use crates_io_docs_rs::RealDocsRsClient;
 use crates_io_env_vars::var;
+use crates_io_fastly::Fastly;
 use crates_io_index::RepositoryConfig;
 use crates_io_og_image::OgImageGenerator;
 use crates_io_team_repo::TeamRepoImpl;
 use crates_io_worker::Runner;
 use object_store::prefix::PrefixStore;
-use reqwest::Client;
 use std::sync::Arc;
 use std::thread::sleep;
 use std::time::Duration;
@@ -79,13 +78,11 @@ fn main() -> anyhow::Result<()> {
     let downloads_archive_store = PrefixStore::new(storage.as_inner(), "archive/version-downloads");
     let downloads_archive_store = Box::new(downloads_archive_store);
 
-    let client = Client::builder()
-        .timeout(Duration::from_secs(45))
-        .build()
-        .expect("Couldn't build client");
-
     let emails = Emails::from_environment(&config);
-    let fastly = Fastly::from_environment(client.clone());
+
+    let fastly_api_token = var("FASTLY_API_TOKEN")?.map(Into::into);
+    let fastly = fastly_api_token.map(Fastly::new);
+
     let team_repo = TeamRepoImpl::default();
 
     let docs_rs = RealDocsRsClient::from_environment().map(|cl| Box::new(cl) as _);

src/bin/background-worker.rs

@@ -27,7 +27,6 @@ pub mod config;
 pub mod controllers;
 pub mod db;
 pub mod email;
-pub mod fastly;
 pub mod headers;
 pub mod index;
 mod licenses;

src/fastly.rs

@@ -1,13 +1,13 @@
 use crate::Emails;
 use crate::cloudfront::CloudFront;
-use crate::fastly::Fastly;
 use crate::storage::Storage;
 use crate::typosquat;
 use crate::worker::jobs::ProcessCloudfrontInvalidationQueue;
 use anyhow::Context;
 use bon::Builder;
 use crates_io_database::models::CloudFrontInvalidationQueueItem;
 use crates_io_docs_rs::DocsRsClient;
+use crates_io_fastly::Fastly;
 use crates_io_index::{Repository, RepositoryConfig};
 use crates_io_og_image::OgImageGenerator;
 use crates_io_team_repo::TeamRepo;
@@ -91,8 +91,13 @@ impl Environment {
             result.context("Failed to enqueue CloudFront invalidation processing job")?;
         }
 
-        if let Some(fastly) = self.fastly() {
-            fastly.invalidate(path).await.context("Fastly")?;
+        if let Some(fastly) = self.fastly()
+            && let Some(cdn_domain) = &self.config.storage.cdn_prefix
+        {
+            fastly
+                .purge_both_domains(cdn_domain, path)
+                .await
+                .context("Fastly")?;
         }
 
         Ok(())

src/lib.rs

@@ -122,7 +122,8 @@ impl BackgroundJob for GenerateOgImage {
 
         // Invalidate Fastly CDN
         if let Some(fastly) = ctx.fastly()
-            && let Err(error) = fastly.invalidate(&og_image_path).await
+            && let Some(cdn_domain) = &ctx.config.storage.cdn_prefix
+            && let Err(error) = fastly.purge_both_domains(cdn_domain, &og_image_path).await
         {
             warn!("Failed to invalidate Fastly CDN for {crate_name}: {error}");
         }