fix: send browser user-agent for HTTP conversions

Author: Salomondiei08

packages/markitdown/src/markitdown/_markitdown.py

@@ -19,6 +19,12 @@
 from ._stream_info import StreamInfo
 from ._uri_utils import parse_data_uri, file_uri_to_path
 
+DEFAULT_HTTP_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+
 from .converters import (
     PlainTextConverter,
     HtmlConverter,
@@ -449,7 +455,8 @@ def convert_uri(
             )
         # HTTP/HTTPS URIs
         elif uri.startswith("http:") or uri.startswith("https:"):
-            response = self._requests_session.get(uri, stream=True)
+            headers = {"User-Agent": DEFAULT_HTTP_USER_AGENT}
+            response = self._requests_session.get(uri, stream=True, headers=headers)
             response.raise_for_status()
             return self.convert_response(
                 response,

packages/markitdown/tests/test_module_misc.py

@@ -552,3 +552,31 @@ def test_markitdown_llm() -> None:
         test()
         print("OK")
     print("All tests passed!")
+
+
+def test_http_uri_uses_browser_user_agent() -> None:
+    markitdown = MarkItDown()
+
+    response = MagicMock()
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/test.html"
+    response.raise_for_status.return_value = None
+
+    expected_result = MagicMock()
+    markitdown.convert_response = MagicMock(return_value=expected_result)
+    markitdown._requests_session.get = MagicMock(return_value=response)
+
+    result = markitdown.convert("https://example.com/test.html")
+
+    assert result is expected_result
+    markitdown._requests_session.get.assert_called_once()
+    _, kwargs = markitdown._requests_session.get.call_args
+    assert kwargs["stream"] is True
+    assert "headers" in kwargs
+    assert kwargs["headers"]["User-Agent"].startswith("Mozilla/5.0")
+    markitdown.convert_response.assert_called_once_with(
+        response,
+        stream_info=None,
+        file_extension=None,
+        url=None,
+    )