From c2d0351d08107ef81e62bda440109ad62e533e77 Mon Sep 17 00:00:00 2001 From: Mieszko Boczkowski Date: Tue, 9 Sep 2025 15:26:56 +0200 Subject: [PATCH 1/5] feat(requests-ca): respect REQUESTS_CA_BUNDLE environment variable Current implementation of `HttpClient` uses requests library to create a `PreparedRequest` and send it, but do not take into consideration the environment variables that `requests` do. This make it impossible to use custom CA certificates together with Airbyte. This PR adds the support of env settings to `HttpClient`. The implementation is doing what `requests` library recommends when working with `PreparedRequests` to properly handle self-signed certificates: https://requests.readthedocs.io/en/latest/user/advanced/#prepared-requests --- .../sources/streams/http/http_client.py | 3 +++ .../sources/streams/http/test_http_client.py | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 63c451c49..216bca9d1 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -545,6 +545,9 @@ def send_request( data=data, ) + env_settings = self._session.merge_environment_settings(request.url, None, None, None, None) + request_kwargs = {**request_kwargs, **env_settings} + response: requests.Response = self._send_with_retry( request=request, request_kwargs=request_kwargs, diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 5cc6d20e4..f904da8af 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1,6 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import logging +import os from datetime import timedelta from unittest.mock import MagicMock, patch @@ -744,3 +745,23 @@ def test_given_different_headers_then_response_is_not_cached(requests_mock): ) assert second_response.json()["test"] == "second response" + +@patch.dict("os.environ", {"REQUESTS_CA_BUNDLE": "/path/to/ca-bundle.crt"}) +def test_send_request_respects_environment_variables(): + """Test that send_request respects REQUESTS_CA_BUNDLE environment variable.""" + http_client = HttpClient( + name="test", + logger=MagicMock(), + ) + + with patch.object(http_client, '_send_with_retry') as mock_send_with_retry: + http_client.send_request( + http_method="GET", + url="https://api.example.com", + request_kwargs={"timeout": 10} + ) + + passed_kwargs = mock_send_with_retry.call_args[1]["request_kwargs"] + + assert "verify" in passed_kwargs + assert passed_kwargs["verify"] == "/path/to/ca-bundle.crt" From 98a792107d9caa462f61363fa054b3ca0f76ae88 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Wed, 17 Sep 2025 07:07:18 +0000 Subject: [PATCH 2/5] Auto-fix lint and format issues --- unit_tests/sources/streams/http/test_http_client.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index f904da8af..57fc59109 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -746,6 +746,7 @@ def test_given_different_headers_then_response_is_not_cached(requests_mock): assert second_response.json()["test"] == "second response" + @patch.dict("os.environ", {"REQUESTS_CA_BUNDLE": "/path/to/ca-bundle.crt"}) def test_send_request_respects_environment_variables(): """Test that send_request respects REQUESTS_CA_BUNDLE environment variable.""" @@ -754,14 +755,12 @@ def test_send_request_respects_environment_variables(): logger=MagicMock(), ) - with patch.object(http_client, '_send_with_retry') as mock_send_with_retry: + with patch.object(http_client, "_send_with_retry") as mock_send_with_retry: http_client.send_request( - http_method="GET", - url="https://api.example.com", - request_kwargs={"timeout": 10} + http_method="GET", url="https://api.example.com", request_kwargs={"timeout": 10} ) - + passed_kwargs = mock_send_with_retry.call_args[1]["request_kwargs"] - + assert "verify" in passed_kwargs assert passed_kwargs["verify"] == "/path/to/ca-bundle.crt" From 8c745d7d18562490b64ea72a1312bd9010e7b3c1 Mon Sep 17 00:00:00 2001 From: Mieszko Boczkowski Date: Thu, 18 Sep 2025 16:35:41 +0200 Subject: [PATCH 3/5] Fix tests --- airbyte_cdk/sources/streams/http/http_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 216bca9d1..fe85fece8 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -545,7 +545,13 @@ def send_request( data=data, ) - env_settings = self._session.merge_environment_settings(request.url, None, None, None, None) + env_settings = self._session.merge_environment_settings( + url=request.url, + proxies=request_kwargs.get("proxies"), + stream=request_kwargs.get("stream"), + verify=request_kwargs.get("verify"), + cert=request_kwargs.get("cert"), + ) request_kwargs = {**request_kwargs, **env_settings} response: requests.Response = self._send_with_retry( From 2c052e98b6c182db1350b3b9c4f7bea947914aa8 Mon Sep 17 00:00:00 2001 From: Mieszko Boczkowski Date: Thu, 18 Sep 2025 16:58:31 +0200 Subject: [PATCH 4/5] Fix format --- unit_tests/sources/streams/http/test_http_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index eaa0e787f..d628a83ce 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -746,6 +746,7 @@ def test_given_different_headers_then_response_is_not_cached(requests_mock): assert second_response.json()["test"] == "second response" + @patch.dict("os.environ", {"REQUESTS_CA_BUNDLE": "/path/to/ca-bundle.crt"}) def test_send_request_respects_environment_variables(): """Test that send_request respects REQUESTS_CA_BUNDLE environment variable.""" @@ -764,6 +765,7 @@ def test_send_request_respects_environment_variables(): assert "verify" in passed_kwargs assert passed_kwargs["verify"] == "/path/to/ca-bundle.crt" + @pytest.mark.usefixtures("mock_sleep") @pytest.mark.parametrize( "response_code, expected_failure_type, error_message, exception_class", From d1919ec53dc693beb86e6584104c11d365e70360 Mon Sep 17 00:00:00 2001 From: Mieszko Boczkowski Date: Thu, 18 Sep 2025 17:33:22 +0200 Subject: [PATCH 5/5] Fix tests --- unit_tests/sources/streams/http/test_http.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index d2bc4071e..7512c3722 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -92,7 +92,10 @@ def test_requests_native_token_authenticator(): def test_request_kwargs_used(mocker, requests_mock): stream = StubBasicReadHttpStream() - request_kwargs = {"cert": None, "proxies": "google.com"} + request_kwargs = { + "cert": None, + "proxies": {"http": "http://example.com", "https": "http://example.com"}, + } mocker.patch.object(stream, "request_kwargs", return_value=request_kwargs) send_mock = mocker.patch.object( stream._http_client._session, "send", wraps=stream._http_client._session.send @@ -101,8 +104,16 @@ def test_request_kwargs_used(mocker, requests_mock): list(stream.read_records(sync_mode=SyncMode.full_refresh)) - stream._http_client._session.send.assert_any_call(ANY, **request_kwargs) assert send_mock.call_count == 1 + call_args = send_mock.call_args_list[0] + call_kwargs = call_args.kwargs + + assert call_kwargs.get("cert") is None + + proxies = call_kwargs.get("proxies") + assert proxies is not None + assert proxies["http"] == "http://example.com" + assert proxies["https"] == "http://example.com" def test_stub_basic_read_http_stream_read_records(mocker):