diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index d14aab7d1c39a..690c03e789a21 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.9.2 +Low-code: Make `default_paginator.page_token_option` optional + ## 0.9.1 Low-code: Fix filtering vars in `InterpolatedRequestInputProvider.eval_request_inputs` diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index b88908aef8950..2ef4fcd9d672a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -81,22 +81,20 @@ class DefaultPaginator(Paginator, JsonSchemaMixin): decoder (Decoder): decoder to decode the response """ - page_size_option: Optional[RequestOption] - page_token_option: RequestOption pagination_strategy: PaginationStrategy config: Config url_base: Union[InterpolatedString, str] options: InitVar[Mapping[str, Any]] decoder: Decoder = JsonDecoder(options={}) _token: Optional[Any] = field(init=False, repr=False, default=None) + page_size_option: Optional[RequestOption] = None + page_token_option: Optional[RequestOption] = None def __post_init__(self, options: Mapping[str, Any]): if self.page_size_option and self.page_size_option.inject_into == RequestOptionType.path: raise ValueError("page_size_option cannot be set in as path") if self.page_size_option and not self.pagination_strategy.get_page_size(): raise ValueError("page_size_option cannot be set if the pagination strategy does not have a page_size") - if self.pagination_strategy.get_page_size() and not self.page_size_option: - raise ValueError("page_size_option must be set if the pagination strategy has a page_size") if isinstance(self.url_base, str): self.url_base = InterpolatedString(string=self.url_base, options=options) @@ -108,7 +106,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin return None def path(self): - if self._token and self.page_token_option.inject_into == RequestOptionType.path: + if self._token and self.page_token_option and self.page_token_option.inject_into == RequestOptionType.path: # Replace url base to only return the path return str(self._token).replace(self.url_base.eval(self.config), "") else: @@ -155,7 +153,7 @@ def reset(self): def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]: options = {} - if self.page_token_option.inject_into == option_type: + if self.page_token_option and self.page_token_option.inject_into == option_type: if option_type != RequestOptionType.path and self._token: options[self.page_token_option.field_name] = self._token if self.page_size_option and self.pagination_strategy.get_page_size() and self.page_size_option.inject_into == option_type: diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index c0c0743d3f077..62934697090b8 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.9.1", + version="0.9.2", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index f6ec277d0a836..8aec567318bbf 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -194,33 +194,11 @@ def test_page_size_option_cannot_be_set_if_strategy_has_no_limit(): pass -def test_page_size_option_must_be_set_if_strategy_has_limit(): - page_size_request_option = None - page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) - cursor_value = "{{ response.next }}" - url_base = "https://airbyte.io" - config = {} - options = {} - strategy = CursorPaginationStrategy(page_size=5, cursor_value=cursor_value, config=config, options=options) - try: - DefaultPaginator( - page_size_option=page_size_request_option, - page_token_option=page_token_request_option, - pagination_strategy=strategy, - config=config, - url_base=url_base, - options={}, - ) - assert False - except ValueError: - pass - - def test_reset(): page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={}) page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) url_base = "https://airbyte.io" config = {} strategy = MagicMock() - DefaultPaginator(page_size_request_option, page_token_request_option, strategy, config, url_base, options={}).reset() + DefaultPaginator(strategy, config, url_base, options={}, page_size_option=page_size_request_option, page_token_option=page_token_request_option).reset() assert strategy.reset.called diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index 5485fab782b94..c7abafc4a9b36 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -547,7 +547,7 @@ def test_generate_schema(): assert default_error_handler["properties"]["backoff_strategies"]["type"] == "array" default_paginator = schema["definitions"]["DefaultPaginator"]["allOf"][1] - assert {"page_token_option", "pagination_strategy", "config", "url_base"}.issubset(default_paginator["required"]) + assert {"pagination_strategy", "config", "url_base"}.issubset(default_paginator["required"]) assert default_paginator["properties"]["page_size_option"]["$ref"] == "#/definitions/RequestOption" assert default_paginator["properties"]["page_token_option"]["$ref"] == "#/definitions/RequestOption" assert {"$ref": "#/definitions/CursorPaginationStrategy"} in default_paginator["properties"]["pagination_strategy"]["anyOf"]