From b973bc8dd3a4a7b9454e3827e7403e4816f92bff Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Thu, 4 May 2023 09:32:40 +0300 Subject: [PATCH] Source Mixpanel: ConnectionResetError fix (#25543) * added HttpAvailabilityStrategy and undeclared fields for annotations and export streams * added extra fields to export stream, updated expected records * added exponential backoff * changed testing range days to 10 * updated expected records, added typeTransformer to exports stream * added changelog * removed HttpAvailabilityStrategy property * added null in type for annotations schema * removed unused imports in base.py * added retry for sream export when ConnectionResetError occurs * added changelog * fixed version in changelog * bump version in metadata.md * auto-bump connector version --------- Co-authored-by: Octavia Squidington III --- .../init-oss/src/main/resources/seed/oss_catalog.json | 2 +- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init-oss/src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-mixpanel/Dockerfile | 2 +- .../connectors/source-mixpanel/metadata.yaml | 2 +- .../source-mixpanel/source_mixpanel/streams/export.py | 8 ++++++++ docs/integrations/sources/mixpanel.md | 1 + 7 files changed, 14 insertions(+), 5 deletions(-) diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json index 2d0e34a02149..8865b1b69e35 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json @@ -18111,7 +18111,7 @@ "sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a", "name": "Mixpanel", "dockerRepository": "airbyte/source-mixpanel", - "dockerImageTag": "0.1.32", + "dockerImageTag": "0.1.33", "documentationUrl": "https://docs.airbyte.com/integrations/sources/mixpanel", "icon": "mixpanel.svg", "sourceType": "api", diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml index 813b8830c131..01f1f0953a9c 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml @@ -1276,7 +1276,7 @@ - name: Mixpanel sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a dockerRepository: airbyte/source-mixpanel - dockerImageTag: 0.1.32 + dockerImageTag: 0.1.33 documentationUrl: https://docs.airbyte.com/integrations/sources/mixpanel icon: mixpanel.svg sourceType: api diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml index 4f6936a93229..c7e665e5922c 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml @@ -9209,7 +9209,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mixpanel:0.1.32" +- dockerImage: "airbyte/source-mixpanel:0.1.33" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mixpanel" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index a744159636be..fa7a0493dbfb 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.32 +LABEL io.airbyte.version=0.1.33 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index 9ae5b4450f67..30ced961919b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -12,7 +12,7 @@ data: connectorSubtype: api connectorType: source definitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a - dockerImageTag: 0.1.32 + dockerImageTag: 0.1.33 dockerRepository: airbyte/source-mixpanel githubIssueLabel: source-mixpanel icon: mixpanel.svg diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py index 8b6a816266c4..a3d6c9bae204 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py @@ -88,6 +88,14 @@ def url_base(self): def path(self, **kwargs) -> str: return "export" + def should_retry(self, response: requests.Response) -> bool: + try: + # trying to parse response to avoid ConnectionResetError and retry if it occurs + self.iter_dicts(response.iter_lines(decode_unicode=True)) + except ConnectionResetError: + return True + return super().should_retry(response) + def iter_dicts(self, lines): """ The incoming stream has to be JSON lines format. diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index 2faa5c92cb94..1efa5cfa5151 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -50,6 +50,7 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------| +| 0.1.33 | 2023-04-25 | [25543](https://github.com/airbytehq/airbyte/pull/25543) | Set should_retry for 104 error in stream export | | 0.1.32 | 2023-04-11 | [25056](https://github.com/airbytehq/airbyte/pull/25056) | Set HttpAvailabilityStrategy, add exponential backoff, streams export and annotations add undeclared fields | | 0.1.31 | 2023-02-13 | [22936](https://github.com/airbytehq/airbyte/pull/22936) | Specified date formatting in specification | | 0.1.30 | 2023-01-27 | [22017](https://github.com/airbytehq/airbyte/pull/22017) | Set `AvailabilityStrategy` for streams explicitly to `None` |