Merge 18a4977 into 9c6df79

Author: ZeJ0hn

haystack/components/fetchers/link_content.py

@@ -22,7 +22,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 DEFAULT_USER_AGENT = f"haystack/LinkContentFetcher/{__version__}"
 
 REQUEST_HEADERS = {
@@ -33,6 +32,25 @@
 }
 
 
+def _merge_headers(*args: dict[str, str]) -> dict[str, str]:
+    """
+    Merge a list of dict using case-insensitively
+
+    :param args: a list of dict to merge
+    :returns: The merged dict
+    """
+    merged = {}
+    keymap = {}
+
+    for d in args:
+        for k, v in d.items():
+            kl = k.lower()
+            keymap[kl] = k
+            merged[kl] = v
+
+    return {keymap[kl]: v for kl, v in merged.items()}
+
+
 def _text_content_handler(response: httpx.Response) -> ByteStream:
     """
     Handles text content.
@@ -169,17 +187,24 @@ def __init__(  # pylint: disable=too-many-positional-arguments
             after=self._switch_user_agent,
         )
         def get_response(url):
-            # Build headers with precedence:
-            # client defaults -> component defaults -> user-provided -> rotating UA
-            base = dict(self._client.headers)
-            headers = {**base, **REQUEST_HEADERS, **self.request_headers}
-            headers["User-Agent"] = self.user_agents[self.current_user_agent_idx]  # rotation wins
-            response = self._client.get(url, headers=headers)
+            response = self._client.get(url, headers=self._get_headers())
             response.raise_for_status()
             return response
 
         self._get_response: Callable = get_response
 
+    def _get_headers(self):
+        """
+        Build headers with precedence
+
+        client defaults -> component defaults -> user-provided -> rotating UA
+        """
+        base = dict(self._client.headers)
+        headers = _merge_headers(
+            base, REQUEST_HEADERS, self.request_headers, {"User-Agent": self.user_agents[self.current_user_agent_idx]}
+        )
+        return headers
+
     def __del__(self):
         """
         Clean up resources when the component is deleted.
@@ -378,10 +403,7 @@ async def _get_response_async(self, url: str, client: httpx.AsyncClient) -> http
 
         while attempt <= self.retry_attempts:
             try:
-                base = dict(client.headers)
-                headers = {**base, **REQUEST_HEADERS, **self.request_headers}
-                headers["User-Agent"] = self.user_agents[self.current_user_agent_idx]
-                response = await client.get(url, headers=headers)
+                response = await client.get(url, headers=self._get_headers())
                 response.raise_for_status()
                 return response
             except (httpx.HTTPStatusError, httpx.RequestError) as e:

haystack/tracing/datadog.py

@@ -31,7 +31,7 @@ def set_tag(self, key: str, value: Any) -> None:
         :param value: the value of the tag.
         """
         coerced_value = tracing_utils.coerce_tag_value(value)
-        self._span.set_tag(key, coerced_value)
+        self._span.set_tag(key, str(coerced_value))
 
     def raw_span(self) -> Any:
         """

releasenotes/notes/fix-bad-request-ont-link-content-f021c7012d63a60e.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Ensure that requests headers keys are unique in link_content