From 19c8efd33b3344508a4bfd4f4a14a477be74681e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 2 Sep 2021 09:40:54 +0300 Subject: [PATCH] :bug: Slack source connector: do not give up retry attempts if there is 429 error (#5708) * Slack source connector: increase retry attempts to 20. --- airbyte-integrations/connectors/source-slack/setup.py | 9 ++++++++- .../connectors/source-slack/source_slack/source.py | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-slack/setup.py b/airbyte-integrations/connectors/source-slack/setup.py index f4c9831e8c42e..3d67eaf21296d 100644 --- a/airbyte-integrations/connectors/source-slack/setup.py +++ b/airbyte-integrations/connectors/source-slack/setup.py @@ -25,12 +25,19 @@ from setuptools import find_packages, setup +TEST_REQUIREMENTS = [ + "pytest~=6.1", +] + setup( name="source_slack", description="Source implementation for Slack.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk", "pytest==6.1.2", "slack_sdk==3.4.2", "pendulum>=2,<3"], + install_requires=["airbyte-cdk~=0.1.13", "slack_sdk==3.4.2", "pendulum>=2,<3"], package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, ) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 81507e1f8669e..d372d4ab67b1c 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -43,6 +43,11 @@ class SlackStream(HttpStream, ABC): primary_key = "id" page_size = 100 + @property + def max_retries(self) -> int: + # Slack's rate limiting can be unpredictable so we increase the max number of retries by a lot before failing + return 20 + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """Slack uses a cursor-based pagination strategy. Extract the cursor from the response if it exists and return it in a format