From 013e11d3105c4e07e274255a45d877b2bcf31458 Mon Sep 17 00:00:00 2001
From: Sohom <sdatta4@ncsu.edu>
Date: Wed, 8 Oct 2025 09:11:53 -0400
Subject: [PATCH] Add a useragent header when fetching data + print response if
 not JSON

This commit introduces two fixes/QoL changes. First off, it adds a
useragent to every request sent by this tool. This is required iff
the remote server rejects requests without any specific useragent
being set (an example for this is Wikimedia Toolforge, which will
drop all HTTP requests without a User-Agent header

See: https://wikitech.wikimedia.org/wiki/Help:Toolforge/Banned#Banned_user_agents_and_IP_addresses

Additionally the second change introduced prints the body of the
response into stderr iff the tool encounters a error in parsing the
JSON data into JWKS sets. This is particularly useful because it
(~~allowed me to debug the error above~~) will help folks figure out
exactly what data got sent back from the server and why the tool might
not be able to validate the keys.
---
 crates/http-signature-directory/src/main.rs | 34 +++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/crates/http-signature-directory/src/main.rs b/crates/http-signature-directory/src/main.rs
index 36957c9..78cb302 100644
--- a/crates/http-signature-directory/src/main.rs
+++ b/crates/http-signature-directory/src/main.rs
@@ -7,7 +7,7 @@ use log::{debug, info};
 use reqwest::{
     Url,
     blocking::Client,
-    header::{ACCEPT, CONTENT_TYPE},
+    header::{ACCEPT, CONTENT_TYPE, USER_AGENT},
 };
 use serde::{Deserialize, Serialize};
 use web_bot_auth::{
@@ -17,12 +17,16 @@ use web_bot_auth::{
 };
 
 const MIME_TYPE: &str = "application/http-message-signatures-directory+json";
+const DEFAULT_USERAGENT: &str = "http-signature-directory-test-script/0.1.0";
 
 #[derive(Parser)]
 #[command(version, about, long_about = None)]
 struct Cli {
     /// URL pointing to your HTTP Message Signature JSON Web Key Set e.g. `https://example.com/.well-known/http-message-signatures-directory`
     url: String,
+    /// Optional useragent that can be used to customize the useragent being sent to the server.
+    #[arg (default_value_t = String::from(DEFAULT_USERAGENT))]
+    user_agent: String,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -130,6 +134,7 @@ fn main() -> Result<(), String> {
     let response = match Client::new()
         .get(cli.url.clone())
         .header(ACCEPT, MIME_TYPE)
+        .header(USER_AGENT, cli.user_agent)
         .send()
     {
         Ok(response) => response,
@@ -189,8 +194,29 @@ fn main() -> Result<(), String> {
         signature_inputs
     );
 
+    let body_text = match response.text() {
+        Ok(text) => text,
+        Err(err) => {
+            let error_msg = format!("Failed to read response body: {:?}", err);
+            errors.push(error_msg.clone());
+            let result = ValidationResult {
+                success: false,
+                message: error_msg.clone(),
+                details: ValidationDetails {
+                    url: cli.url.clone(),
+                    keys_count: 0,
+                    validated_keys: vec![],
+                    errors,
+                    warnings,
+                },
+            };
+            eprintln!("{}", serde_json::to_string_pretty(&result).unwrap());
+            return Err("Body read failed".to_string());
+        }
+    };
+
     // Parse JSON Web Key Set
-    let json_web_key_set: JSONWebKeySet = match response.json() {
+    let json_web_key_set: JSONWebKeySet = match serde_json::from_str(&body_text) {
         Ok(jwks) => jwks,
         Err(error) => {
             let error_msg = format!("Failed to parse content as JSON web key set: {:?}", error);
@@ -207,6 +233,10 @@ fn main() -> Result<(), String> {
                 },
             };
             eprintln!("{}", serde_json::to_string_pretty(&result).unwrap());
+            eprintln!("This was the data the server sent back:");
+            eprintln!("--- RAW RESPONSE BODY START ---");
+            println!("{}", body_text);
+            eprintln!("--- RAW RESPONSE BODY END ---");
             return Err("JSON parsing failed".to_string());
         }
     };