autofix parameter for invalid links on paged responses

CHANGELOG.md

@@ -9,17 +9,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Add `autofix_paging` parameter to `StacApiIO` to allow auto-fix of invalid links returned by servers when iterating paged responses ([#852](https://github.com/stac-utils/pystac-client/pull/852)).
 - Add comprehensive test coverage for warning context managers (`ignore()` and `strict()`) ([#832](https://github.com/stac-utils/pystac-client/pull/832))
 - Moved -Werror to pyproject.toml ([#841](https://github.com/stac-utils/pystac-client/pull/841))
+- Add comprehensive test coverage for ConformanceClasses enum ([#834](https://github.com/stac-utils/pystac-client/pull/834))
 
 ### Changed
 
 - Make `get_collection` raise if `collection_id` is empty ([#809](https://github.com/stac-utils/pystac-client/pull/809))
 
-### Added
-
-- Add comprehensive test coverage for ConformanceClasses enum ([#834](https://github.com/stac-utils/pystac-client/pull/834))
-
 ### Documentation
 
 - Update contributing guide to consistently use `uv` workflow ([#822](https://github.com/stac-utils/pystac-client/pull/822))

pystac_client/stac_api_io.py

@@ -47,6 +47,7 @@ def __init__(
         request_modifier: Callable[[Request], Request | None] | None = None,
         timeout: Timeout | None = None,
         max_retries: int | Retry | None = 5,
+        autofix_paging: bool = True,
     ):
         """Initialize class for API IO
 
@@ -69,11 +70,15 @@ def __init__(
               <https://requests.readthedocs.io/en/latest/api/#main-interface>`__.
             max_retries: The number of times to retry requests. Set to ``None`` to
               disable retries.
+            autofix_paging: Whether to attempt automatically fixing paging issues
+              that can be identified from paged responses links mismatching expectations
+              compared to the original request.
 
         Return:
             StacApiIO : StacApiIO instance
         """
-        # TODO - this should super() to parent class
+        super().__init__(headers)
+        self.autofix_paging = autofix_paging
 
         if conformance is not None:
             warnings.warn(
@@ -311,7 +316,29 @@ def get_pages(
         )
         while next_link:
             link = Link.from_dict(next_link)
-            page = self.read_json(link, parameters=parameters)
+            try:
+                page = self.read_json(link, parameters=parameters)
+            except APIError as exc:
+                # retry with fixes if enabled and changes could be identified
+                params = link.to_dict()
+                meth = params.get("method")
+                if link.href == url and meth == method:
+                    raise  # other unidentified error
+                if self.autofix_paging:
+                    logger.warning(
+                        "Error retrieving paged results from 'next' link "
+                        f"due to incompatible URL {link.href} or HTTP {meth} method. "
+                        "Retrying with original request parameters."
+                    )
+                    params["method"] = method
+                    page = self.read_json(url, method=method, parameters=params)
+                else:
+                    logger.error(
+                        "Error retrieving paged results from 'next' link "
+                        f"due to incompatible URL {link.href} or HTTP {meth} method.",
+                        exc_info=exc,
+                    )
+                    raise
             if not (page.get("features") or page.get("collections")):
                 return None
             yield page

tests/test_item_search.py

@@ -11,7 +11,9 @@
 from requests_mock import Mocker
 
 from pystac_client import Client
+from pystac_client.exceptions import APIError
 from pystac_client.item_search import ItemSearch
+from pystac_client.stac_api_io import StacApiIO
 
 from .helpers import STAC_URLS, read_data_file
 
@@ -240,6 +242,39 @@ def test_result_paging_max_items(self) -> None:
         assert num_pages == 3
         assert len(items) == 25
 
+    @pytest.mark.vcr
+    def test_result_paging_bad_next_link_autofix_enabled(self, vcr) -> None:
+        search = ItemSearch(
+            url=SEARCH_URL,
+            method="POST",
+            collections="naip",
+            limit=1,
+        )
+        num_pages = 0
+        items = list()
+        for page in search.pages_as_dicts():
+            num_pages += 1
+            items.extend(page["features"])
+        assert num_pages == 2
+        assert len(items) == 2
+        assert vcr.play_count == 3, (
+            "should have made the 2 valid requests and 1 autofix request"
+        )
+
+    @pytest.mark.vcr
+    def test_result_paging_bad_next_link_autofix_disabled(self, vcr) -> None:
+        search = ItemSearch(
+            url=SEARCH_URL,
+            method="POST",
+            collections="naip",
+            limit=1,
+            stac_io=StacApiIO(autofix_paging=False),
+        )
+        with pytest.raises(APIError, match="Method not allowed"):
+            for _ in search.pages_as_dicts():
+                pass
+        assert vcr.play_count == 2
+
     @pytest.mark.vcr
     def test_item_collection(self) -> None:
         search = ItemSearch(