🐛 Source Slack: Fix max retries issue (#5697)

* Increased max retries to 10 * Add docs * Increased timeout seconds for acceptance tests * Fixed for pass tests * Updated to default backoff * Add sample config
airbytehq · Aug 31, 2021 · ff70fd3 · ff70fd3
1 parent 7584440
commit ff70fd3
Show file tree

Hide file tree

Showing 9 changed files with 17 additions and 24 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "c2281cee-86f9-4a86-bb48-d23286b4c7bd",
   "name": "Slack",
   "dockerRepository": "airbyte/source-slack",
-  "dockerImageTag": "0.1.9",
+  "dockerImageTag": "0.1.10",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/slack",
   "icon": "slack.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -349,7 +349,7 @@
 - sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd
   name: Slack
   dockerRepository: airbyte/source-slack
-  dockerImageTag: 0.1.9
+  dockerImageTag: 0.1.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/slack
   icon: slack.svg
 - sourceDefinitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1

diff --git a/airbyte-integrations/connectors/source-slack/Dockerfile b/airbyte-integrations/connectors/source-slack/Dockerfile
@@ -16,5 +16,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.9
+LABEL io.airbyte.version=0.1.10
 LABEL io.airbyte.name=airbyte/source-slack
diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml
@@ -12,13 +12,15 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/full_refresh_catalog.json"
+      timeout_seconds: 7200
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      timeout_seconds: 3600
+      timeout_seconds: 7200
       cursor_paths:
         channel_messages: ["float_ts"]
         threads: ["float_ts"]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/full_refresh_catalog.json"
+      timeout_seconds: 7200
diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/full_refresh_catalog.json b/airbyte-integrations/connectors/source-slack/integration_tests/full_refresh_catalog.json
@@ -9,15 +9,6 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
-    {
-      "stream": {
-        "name": "channel_members",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
     {
       "stream": {
         "name": "users",

diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-slack/integration_tests/invalid_config.json
@@ -1,3 +1,6 @@
 {
-  "api_token": "wrong.api.key123"
+  "api_token": "invalid_api_token",
+  "start_date": "2022-07-22T20:00:00Z",
+  "lookback_window": 2,
+  "join_channels": true
 }
diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-slack/integration_tests/sample_config.json
@@ -0,0 +1,6 @@
+{
+  "api_token": "api_token",
+  "start_date": "2021-07-22T20:00:00Z",
+  "lookback_window": 2,
+  "join_channels": true
+}
diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py
@@ -74,16 +74,6 @@ def parse_response(
         json_response = response.json()
         yield from json_response.get(self.data_field, [])
 
-    def backoff_time(self, response: requests.Response) -> Optional[float]:
-        """This method is called if we run into the rate limit.
-        Slack puts the retry time in the `Retry-After` response header so we
-        we return that value. If the response is anything other than a 429 (e.g: 5XX)
-        fall back on default retry behavior.
-
-        Rate Limits Docs: https://api.slack.com/docs/rate-limits#web"""
-
-        return int(response.headers.get("Retry-After", 0))
-
     @property
     @abstractmethod
     def data_field(self) -> str:

diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md
@@ -101,6 +101,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.1.10   | 2021-08-27 | [5697](https://github.com/airbytehq/airbyte/pull/5697) | Fixed max retries issue |
 | 0.1.9   | 2021-07-20 | [4860](https://github.com/airbytehq/airbyte/pull/4860) | Fixed reading threads issue |
 | 0.1.8   | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float_ts primary key |
 | 0.1.7   | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector |