Add User-Agent header to fetch request

src/bsky.rs

@@ -1,4 +1,5 @@
 use crate::client::{ClientInfo, FetchClient};
+use crate::fetch;
 use crate::hatebu::Entry;
 use atrium_api::app::bsky::embed::external;
 use atrium_api::app::bsky::feed::post::RecordEmbedRefs;
@@ -12,7 +13,7 @@ use atrium_api::types::Union;
 use http::Uri;
 use std::sync::{Arc, RwLock};
 use webpage::HTML;
-use worker::{Fetch, Url};
+use worker::Url;
 
 pub(crate) struct BskyAgent {
     client: AtpServiceClient<FetchClient>,
@@ -98,7 +99,7 @@ impl BskyAgent {
                 }
                 other => other,
             }?;
-            let data = Fetch::Url(url).send().await?.bytes().await?;
+            let data = fetch(url.as_str()).await?.bytes().await?;
             let uploaded = self
                 .client
                 .service

src/lib.rs

@@ -9,8 +9,8 @@ use markup5ever_rcdom::{Handle, NodeData, RcDom};
 use mime::Mime;
 use std::io::Cursor;
 use webpage::HTML;
-use worker::Env;
-use worker::{console_error, console_log, event, Fetch};
+use worker::{console_error, console_log, event, Fetch, Method, Response};
+use worker::{Env, Request};
 use worker::{ScheduleContext, ScheduledEvent};
 
 const KV_NAMESPACE: &str = "kv";
@@ -69,7 +69,7 @@ async fn post2bsky(
 }
 
 async fn get_webpage(url: &str) -> Result<HTML, Box<dyn std::error::Error>> {
-    let mut res = Fetch::Url(url.parse()?).send().await?;
+    let mut res = fetch(url).await?;
     let content_type = res
         .headers()
         .get(http::header::CONTENT_TYPE.as_str())?
@@ -131,3 +131,9 @@ fn extract_charset(handle: &Handle) -> Option<String> {
     }
     None
 }
+
+pub async fn fetch(uri: &str) -> Result<Response, Box<dyn std::error::Error>> {
+    let mut req = Request::new(uri, Method::Get)?;
+    req.headers_mut()?.set("User-Agent", "Cloudflare Workers")?;
+    Ok(Fetch::Request(req).send().await?)
+}