From a088206d9870fdbfd4e96fd60a5d2817be11f475 Mon Sep 17 00:00:00 2001 From: Baz Date: Tue, 27 Sep 2022 15:42:35 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Freshdesk:=20skip=20the?= =?UTF-8?q?=20stream=20if=20`forbidden`=20for=20current=20subscription=20p?= =?UTF-8?q?lan=20(#17243)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-freshdesk/Dockerfile | 2 +- .../source_freshdesk/streams.py | 20 +++++++++++++++---- docs/integrations/sources/freshdesk.md | 1 + 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 079e4cf0b701a..84959314f51d7 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -315,7 +315,7 @@ - name: Freshdesk sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 dockerRepository: airbyte/source-freshdesk - dockerImageTag: 0.3.3 + dockerImageTag: 0.3.4 documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk icon: freshdesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 50783a94d3658..c14ec571b6d69 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3230,7 +3230,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-freshdesk:0.3.3" +- dockerImage: "airbyte/source-freshdesk:0.3.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-freshdesk/Dockerfile b/airbyte-integrations/connectors/source-freshdesk/Dockerfile index 46bff3cef69c2..8bcdb0fb5f494 100644 --- a/airbyte-integrations/connectors/source-freshdesk/Dockerfile +++ b/airbyte-integrations/connectors/source-freshdesk/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshdesk ./source_freshdesk ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.3 +LABEL io.airbyte.version=0.3.4 LABEL io.airbyte.name=airbyte/source-freshdesk diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 9c88fd59b6c6a..f4734c50e8455 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -24,6 +24,8 @@ class FreshdeskStream(HttpStream, ABC): result_return_limit = 100 primary_key = "id" link_regex = re.compile(r'<(.*?)>;\s*rel="next"') + raise_on_http_errors = True + forbidden_stream = False def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): super().__init__(authenticator=authenticator) @@ -44,6 +46,16 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: if response.status_code == requests.codes.too_many_requests: return float(response.headers.get("Retry-After", 0)) + def should_retry(self, response: requests.Response) -> bool: + if isinstance(response.json(), dict): + if response.status_code == requests.codes.FORBIDDEN and response.json().get("code") == "require_feature": + self.forbidden_stream = True + setattr(self, "raise_on_http_errors", False) + self.logger.warn(f"Stream `{self.name}` is not available. {response.json().get('message')}") + else: + return super().should_retry(response) + return super().should_retry(response) + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: link_header = response.headers.get("Link") if not link_header: @@ -57,7 +69,7 @@ def parse_link_params(self, link_query_params: Mapping[str, List[str]]) -> Mappi return {"per_page": link_query_params["per_page"][0], "page": link_query_params["page"][0]} def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: params = {"per_page": self.result_return_limit} if next_page_token and "page" in next_page_token: @@ -83,7 +95,7 @@ def read_records( def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: data = response.json() - return data if data else [] + return {} if self.forbidden_stream else data if data else [] class IncrementalFreshdeskStream(FreshdeskStream, IncrementalMixin): @@ -108,7 +120,7 @@ def state(self, value: MutableMapping[str, Any]): self._cursor_value = value.get(self.cursor_field, self.start_date) def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) params[self.cursor_filter] = stream_state.get(self.cursor_field, self.start_date) @@ -301,7 +313,7 @@ def path(self, **kwargs) -> str: return "tickets" def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) includes = ["description", "requester", "stats"] diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md index 34b10d984c888..edf956983df3a 100644 --- a/docs/integrations/sources/freshdesk.md +++ b/docs/integrations/sources/freshdesk.md @@ -82,6 +82,7 @@ The Freshdesk connector should not run into Freshdesk API limitations under norm | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------|:-------------------------------------------------------------------------------| +| 0.3.4 | 2022-09-27 | [17243](https://github.com/airbytehq/airbyte/pull/17243) | Fixed the issue, when selected stream is not available due to Subscription Plan. | 0.3.3 | 2022-08-06 | [15378](https://github.com/airbytehq/airbyte/pull/15378) | Allow backward campatibility for input configuration | 0.3.2 | 2022-06-23 | [14049](https://github.com/airbytehq/airbyte/pull/14049) | Update parsing of start_date | | 0.3.1 | 2022-06-03 | [13332](https://github.com/airbytehq/airbyte/pull/13332) | Add new streams |