CDK: make default_paginator.page_token_option optional (#19368)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.9.2
+Low-code: Make `default_paginator.page_token_option` optional
+
 ## 0.9.1
 Low-code: Fix filtering vars in `InterpolatedRequestInputProvider.eval_request_inputs`
 

airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -81,22 +81,20 @@ class DefaultPaginator(Paginator, JsonSchemaMixin):
         decoder (Decoder): decoder to decode the response
     """
 
-    page_size_option: Optional[RequestOption]
-    page_token_option: RequestOption
     pagination_strategy: PaginationStrategy
     config: Config
     url_base: Union[InterpolatedString, str]
     options: InitVar[Mapping[str, Any]]
     decoder: Decoder = JsonDecoder(options={})
     _token: Optional[Any] = field(init=False, repr=False, default=None)
+    page_size_option: Optional[RequestOption] = None
+    page_token_option: Optional[RequestOption] = None
 
     def __post_init__(self, options: Mapping[str, Any]):
         if self.page_size_option and self.page_size_option.inject_into == RequestOptionType.path:
             raise ValueError("page_size_option cannot be set in as path")
         if self.page_size_option and not self.pagination_strategy.get_page_size():
             raise ValueError("page_size_option cannot be set if the pagination strategy does not have a page_size")
-        if self.pagination_strategy.get_page_size() and not self.page_size_option:
-            raise ValueError("page_size_option must be set if the pagination strategy has a page_size")
         if isinstance(self.url_base, str):
             self.url_base = InterpolatedString(string=self.url_base, options=options)
 
@@ -108,7 +106,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin
             return None
 
     def path(self):
-        if self._token and self.page_token_option.inject_into == RequestOptionType.path:
+        if self._token and self.page_token_option and self.page_token_option.inject_into == RequestOptionType.path:
             # Replace url base to only return the path
             return str(self._token).replace(self.url_base.eval(self.config), "")
         else:
@@ -155,7 +153,7 @@ def reset(self):
 
     def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]:
         options = {}
-        if self.page_token_option.inject_into == option_type:
+        if self.page_token_option and self.page_token_option.inject_into == option_type:
             if option_type != RequestOptionType.path and self._token:
                 options[self.page_token_option.field_name] = self._token
         if self.page_size_option and self.pagination_strategy.get_page_size() and self.page_size_option.inject_into == option_type:

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.9.1",
+    version="0.9.2",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py

@@ -194,33 +194,11 @@ def test_page_size_option_cannot_be_set_if_strategy_has_no_limit():
         pass
 
 
-def test_page_size_option_must_be_set_if_strategy_has_limit():
-    page_size_request_option = None
-    page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={})
-    cursor_value = "{{ response.next }}"
-    url_base = "https://airbyte.io"
-    config = {}
-    options = {}
-    strategy = CursorPaginationStrategy(page_size=5, cursor_value=cursor_value, config=config, options=options)
-    try:
-        DefaultPaginator(
-            page_size_option=page_size_request_option,
-            page_token_option=page_token_request_option,
-            pagination_strategy=strategy,
-            config=config,
-            url_base=url_base,
-            options={},
-        )
-        assert False
-    except ValueError:
-        pass
-
-
 def test_reset():
     page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={})
     page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={})
     url_base = "https://airbyte.io"
     config = {}
     strategy = MagicMock()
-    DefaultPaginator(page_size_request_option, page_token_request_option, strategy, config, url_base, options={}).reset()
+    DefaultPaginator(strategy, config, url_base, options={}, page_size_option=page_size_request_option, page_token_option=page_token_request_option).reset()
     assert strategy.reset.called

airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py

@@ -547,7 +547,7 @@ def test_generate_schema():
     assert default_error_handler["properties"]["backoff_strategies"]["type"] == "array"
 
     default_paginator = schema["definitions"]["DefaultPaginator"]["allOf"][1]
-    assert {"page_token_option", "pagination_strategy", "config", "url_base"}.issubset(default_paginator["required"])
+    assert {"pagination_strategy", "config", "url_base"}.issubset(default_paginator["required"])
     assert default_paginator["properties"]["page_size_option"]["$ref"] == "#/definitions/RequestOption"
     assert default_paginator["properties"]["page_token_option"]["$ref"] == "#/definitions/RequestOption"
     assert {"$ref": "#/definitions/CursorPaginationStrategy"} in default_paginator["properties"]["pagination_strategy"]["anyOf"]