Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Changed

- `tilebox-storage`: Replaced `httpx` with `niquests` for ASF HTTP downloads.

## [0.50.1] - 2026-04-01

### Added
Expand Down
4 changes: 2 additions & 2 deletions tilebox-storage/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ classifiers = [
requires-python = ">=3.10"
dependencies = [
"tilebox-datasets",
"httpx>=0.27",
"niquests>=3.18",
"aiofile>=3.8",
"folium>=0.15",
"shapely>=2",
Expand All @@ -33,10 +33,10 @@ dependencies = [
[dependency-groups]
dev = [
"hypothesis>=6.112.1",
"pytest-httpx>=0.30.0",
"pytest-asyncio>=0.24.0",
"pytest-cov>=5.0.0",
"pytest>=8.3.2",
"responses>=0.26.0",
]

[project.urls]
Expand Down
180 changes: 180 additions & 0 deletions tilebox-storage/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,186 @@
from collections.abc import Iterator
from inspect import isawaitable
from io import BytesIO
from sys import modules
from typing import Any
from unittest import mock as std_mock

import niquests
import niquests.adapters as niquests_adapters
import niquests.exceptions as niquests_exceptions
import niquests.models as niquests_models
import pytest
import requests.compat as requests_compat
from niquests.packages import urllib3

modules["requests"] = niquests
modules["requests.adapters"] = niquests_adapters
modules["requests.models"] = niquests_models
modules["requests.exceptions"] = niquests_exceptions
modules["requests.packages.urllib3"] = urllib3
modules["requests.compat"] = requests_compat

import responses # noqa: E402


# see https://niquests.readthedocs.io/en/latest/community/extensions.html#responses
class _TransferState:
def __init__(self) -> None:
self.data_in_count = 0


class _AsyncRawBody:
def __init__(self, body: bytes) -> None:
self._body = BytesIO(body)
self._fp = _TransferState()

async def read(self, chunk_size: int = -1, decode_content: bool = True) -> bytes:
_ = decode_content
chunk = self._body.read() if chunk_size == -1 else self._body.read(chunk_size)
self._fp.data_in_count += len(chunk)
return chunk

async def close(self) -> None:
self._body.close()

def release_conn(self) -> None:
return None


class NiquestsMock(responses.RequestsMock):
"""Extend responses to patch Niquests' sync and async adapters."""

def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, target="niquests.adapters.HTTPAdapter.send", **kwargs)
self._patcher_async: Any | None = None

def unbound_on_async_send(self) -> Any:
async def send(adapter: Any, request: Any, *args: Any, **kwargs: Any) -> Any:
if args:
try:
kwargs["stream"] = args[0]
kwargs["timeout"] = args[1]
kwargs["verify"] = args[2]
kwargs["cert"] = args[3]
kwargs["proxies"] = args[4]
except IndexError:
pass

stream = bool(kwargs.get("stream"))
resp = self._on_request(adapter, request, **kwargs)

if stream:
body = getattr(getattr(resp, "raw", None), "read", lambda: getattr(resp, "_content", b""))()
if isawaitable(body):
body = await body
if body is None or isinstance(body, bool):
body = b""
if isinstance(body, str):
body = body.encode()
resp.__class__ = niquests.AsyncResponse
resp.raw = _AsyncRawBody(body)
return resp

resp.__class__ = niquests.Response
return resp

return send

def unbound_on_send(self) -> Any:
def send(adapter: Any, request: Any, *args: Any, **kwargs: Any) -> Any:
if args:
try:
kwargs["stream"] = args[0]
kwargs["timeout"] = args[1]
kwargs["verify"] = args[2]
kwargs["cert"] = args[3]
kwargs["proxies"] = args[4]
except IndexError:
pass

return self._on_request(adapter, request, **kwargs)

return send

def start(self) -> None:
if self._patcher:
return

self._patcher = std_mock.patch(target=self.target, new=self.unbound_on_send())
self._patcher_async = std_mock.patch(
target=self.target.replace("HTTPAdapter", "AsyncHTTPAdapter"),
new=self.unbound_on_async_send(),
)
self._patcher.start()
self._patcher_async.start()

def stop(self, allow_assert: bool = True) -> None:
if self._patcher:
self._patcher.stop()
if self._patcher_async is not None:
self._patcher_async.stop()
self._patcher = None
self._patcher_async = None

if not self.assert_all_requests_are_fired or not allow_assert:
return

not_called = [match for match in self.registered() if match.call_count == 0]
if not_called:
raise AssertionError(
f"Not all requests have been executed {[(match.method, match.url) for match in not_called]!r}"
)


mock = _default_mock = NiquestsMock(assert_all_requests_are_fired=False)
responses.mock = mock
responses._default_mock = _default_mock
for kw in [
"activate",
"add",
"_add_from_file",
"add_callback",
"add_passthru",
"assert_call_count",
"calls",
"delete",
"DELETE",
"get",
"GET",
"head",
"HEAD",
"options",
"OPTIONS",
"patch",
"PATCH",
"post",
"POST",
"put",
"PUT",
"registered",
"remove",
"replace",
"reset",
"response_callback",
"start",
"stop",
"upsert",
]:
if hasattr(responses, kw):
setattr(responses, kw, getattr(mock, kw))


@pytest.fixture
def anyio_backend() -> str:
return "asyncio"


@pytest.fixture
def responses_mock() -> Iterator[responses.RequestsMock]:
responses.mock.reset()
responses.mock.start()
try:
yield responses.mock
finally:
responses.mock.stop()
responses.mock.reset()
31 changes: 20 additions & 11 deletions tilebox-storage/tests/test_providers.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,36 @@
import re
from typing import cast

import pytest
from httpx import AsyncClient, BasicAuth
from pytest_httpx import HTTPXMock
import responses
from niquests import AsyncSession
from niquests.cookies import RequestsCookieJar

from tilebox.storage.providers import _asf_login

pytestmark = pytest.mark.usefixtures("responses_mock")

ASF_LOGIN_URL = "https://urs.earthdata.nasa.gov/oauth/authorize"


@pytest.mark.asyncio
async def test_asf_login(httpx_mock: HTTPXMock) -> None:
httpx_mock.add_response(headers={"Set-Cookie": "logged_in=yes"})
async def test_asf_login() -> None:
responses.add(responses.GET, ASF_LOGIN_URL, headers={"Set-Cookie": "logged_in=yes"})

client = await _asf_login(("username", "password"))
assert isinstance(client, AsyncClient)
assert "asf_search" in client.headers["Client-Id"]
assert isinstance(client.auth, BasicAuth)
assert client.cookies["logged_in"] == "yes"
cookies = cast(RequestsCookieJar, client.cookies)

await client.aclose()
assert isinstance(client, AsyncSession)
assert "asf_search" in str(client.headers["Client-Id"])
assert client.auth == ("username", "password")
assert cookies["logged_in"] == "yes"

await client.close()


@pytest.mark.asyncio
async def test_asf_login_invalid_auth(httpx_mock: HTTPXMock) -> None:
httpx_mock.add_response(401)
async def test_asf_login_invalid_auth() -> None:
responses.add(responses.GET, ASF_LOGIN_URL, status=401)

with pytest.raises(ValueError, match=re.escape("Invalid username or password.")):
await _asf_login(("username", "password"))
Loading
Loading