From ec39ea71d77d16fd3d7cd1fbd5110444961264ba Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Thu, 31 Aug 2023 12:57:52 +0300 Subject: [PATCH 1/8] #1772 source mixpanel: fix typing --- .../source_mixpanel/streams/engage.py | 3 +- .../unit_tests/test_streams.py | 44 ++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py index 46aa7593d9493..9a52b847f09a4 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py @@ -184,7 +184,8 @@ def get_json_schema(self) -> Mapping[str, Any]: types = { "boolean": {"type": ["null", "boolean"]}, "number": {"type": ["null", "number"], "multipleOf": 1e-20}, - "datetime": {"type": ["null", "string"], "format": "date-time"}, + # no format specified as values can be "2021-12-16T00:00:00", "1638298874", "15/08/53895" + "datetime": {"type": ["null", "string"]}, "object": {"type": ["null", "object"], "additionalProperties": True}, "list": {"type": ["null", "array"], "required": False, "items": {}}, "string": {"type": ["null", "string"]}, diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index cde2bbefb3799..561d90389ad04 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -279,23 +279,47 @@ def engage_schema_response(): 200, { "results": { - "$browser": {"count": 124, "type": "string"}, - "$browser_version": {"count": 124, "type": "string"}, "$created": {"count": 124, "type": "string"}, + "$is_active": {"count": 412, "type": "boolean"}, + "$CreatedDateTimestamp": {"count": 300, "type": "number"}, + "$CreatedDate": {"count": 11, "type": "datetime"}, + "$properties": {"count": 2, "type": "object"}, + "$tags": {"count": 131, "type": "list"} } }, ) def test_engage_schema(requests_mock, engage_schema_response, config): - - stream = EngageSchema(authenticator=MagicMock(), **config) - requests_mock.register_uri("GET", get_url_to_mock(stream), engage_schema_response) - - records = stream.read_records(sync_mode=SyncMode.full_refresh) - - records_length = sum(1 for _ in records) - assert records_length == 3 + stream = Engage(authenticator=MagicMock(), **config) + requests_mock.register_uri("GET", get_url_to_mock(EngageSchema(authenticator=MagicMock(), **config)), engage_schema_response) + assert stream.get_json_schema() == { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": { + "CreatedDate": {"type": ["null", "string"]}, + "CreatedDateTimestamp": {"multipleOf": 1e-20, "type": ["null", "number"]}, + "browser": {"type": ["null", "string"]}, + "browser_version": {"type": ["null", "string"]}, + "city": {"type": ["null", "string"]}, + "country_code": {"type": ["null", "string"]}, + "created": {"type": ["null", "string"]}, + "distinct_id": {"type": ["null", "string"]}, + "email": {"type": ["null", "string"]}, + "first_name": {"type": ["null", "string"]}, + "id": {"type": ["null", "string"]}, + "is_active": {"type": ["null", "boolean"]}, + "last_name": {"type": ["null", "string"]}, + "last_seen": {"format": "date-time", "type": ["null", "string"]}, + "name": {"type": ["null", "string"]}, + "properties": {"additionalProperties": True, "type": ["null", "object"]}, + "region": {"type": ["null", "string"]}, + "tags": {"items": {}, "required": False, "type": ["null", "array"]}, + "timezone": {"type": ["null", "string"]}, + "unblocked": {"type": ["null", "string"]} + }, + "type": "object" + } def test_update_engage_schema(requests_mock, config): From f29f0b0c0a58d76fec05442f6cfbf1633a47d467 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 27 Sep 2023 17:20:03 +0300 Subject: [PATCH 2/8] Fix export primary key --- .../integration_tests/configured_catalog_incremental.json | 4 ++-- .../connectors/source-mixpanel/metadata.yaml | 5 +++++ .../source-mixpanel/source_mixpanel/streams/export.py | 2 +- docs/integrations/sources/mixpanel-migrations.md | 5 +++++ 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 docs/integrations/sources/mixpanel-migrations.md diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json index 13e1c2ae450f6..453ec0d2631fc 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json @@ -21,12 +21,12 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["time"], - "source_defined_primary_key": [["distinct_id"]] + "source_defined_primary_key": [["distinct_id"], ["event"], ["time"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", "cursor_field": ["time"], - "primary_key": [["distinct_id"]] + "primary_key": [["distinct_id"], ["event"], ["time"]] }, { "stream": { diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index e8494fa4a9815..bb9574f031e24 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -18,6 +18,11 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data before updating. + upgradeDeadline: "2023-10-31" suggestedStreams: streams: - export diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py index 05b8d636625a7..d35e428a82f4e 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py @@ -75,7 +75,7 @@ class Export(DateSlicesMixin, IncrementalMixpanelStream): 3 queries per second and 60 queries per hour. """ - primary_key: str = "distinct_id" + primary_key: str = ["distinct_id", "event", "time"] cursor_field: str = "time" transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) diff --git a/docs/integrations/sources/mixpanel-migrations.md b/docs/integrations/sources/mixpanel-migrations.md new file mode 100644 index 0000000000000..7d2a76dcca0d9 --- /dev/null +++ b/docs/integrations/sources/mixpanel-migrations.md @@ -0,0 +1,5 @@ +# Mixpanel Migration Guide + +## Upgrading to 1.0.0 + +In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data before updating. \ No newline at end of file From 0103f3b10a2a087c6f1eb0c7694e98750595331d Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 27 Sep 2023 18:41:36 +0300 Subject: [PATCH 3/8] Update version --- .../connectors/source-mixpanel/Dockerfile | 2 +- .../connectors/source-mixpanel/metadata.yaml | 2 +- docs/integrations/sources/mixpanel.md | 87 ++++++++++--------- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index d4de3e1770967..99c4dfa0e603c 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.41 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index bb9574f031e24..f537d8018b70e 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a - dockerImageTag: 0.1.41 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-mixpanel githubIssueLabel: source-mixpanel icon: mixpanel.svg diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index 3fab41988ae34..06af62fb3aa49 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -49,46 +49,47 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.1.41 | 2023-09-26 | [30149](https://github.com/airbytehq/airbyte/pull/30149) | Change config schema; set checkpointing interval; add suggested streams; add cating datetime fields. | -| 0.1.40 | 2022-09-20 | [30090](https://github.com/airbytehq/airbyte/pull/30090) | Handle 400 error when the credentials become expired | -| 0.1.39 | 2023-09-15 | [30469](https://github.com/airbytehq/airbyte/pull/30469) | Add default primary key `distinct_id` to `Export` stream | -| 0.1.38 | 2023-08-31 | [30028](https://github.com/airbytehq/airbyte/pull/30028) | Handle gracefully project timezone mismatch | -| 0.1.37 | 2023-07-20 | [27932](https://github.com/airbytehq/airbyte/pull/27932) | Fix spec: change start/end date format to `date` | -| 0.1.36 | 2023-06-27 | [27752](https://github.com/airbytehq/airbyte/pull/27752) | Partially revert version 0.1.32; Use exponential backoff; | -| 0.1.35 | 2023-06-12 | [27252](https://github.com/airbytehq/airbyte/pull/27252) | Add should_retry False for 402 error | -| 0.1.34 | 2023-05-15 | [21837](https://github.com/airbytehq/airbyte/pull/21837) | Add "insert_id" field to "export" stream schema | -| 0.1.33 | 2023-04-25 | [25543](https://github.com/airbytehq/airbyte/pull/25543) | Set should_retry for 104 error in stream export | -| 0.1.32 | 2023-04-11 | [25056](https://github.com/airbytehq/airbyte/pull/25056) | Set HttpAvailabilityStrategy, add exponential backoff, streams export and annotations add undeclared fields | -| 0.1.31 | 2023-02-13 | [22936](https://github.com/airbytehq/airbyte/pull/22936) | Specified date formatting in specification | -| 0.1.30 | 2023-01-27 | [22017](https://github.com/airbytehq/airbyte/pull/22017) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.29 | 2022-11-02 | [18846](https://github.com/airbytehq/airbyte/pull/18846) | For "export" stream make line parsing more robust | -| 0.1.28 | 2022-10-06 | [17699](https://github.com/airbytehq/airbyte/pull/17699) | Fix discover step issue cursor field None | -| 0.1.27 | 2022-09-29 | [17415](https://github.com/airbytehq/airbyte/pull/17415) | Disable stream "cohort_members" on discover if not access | -| 0.1.26 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | -| 0.1.25 | 2022-09-27 | [17145](https://github.com/airbytehq/airbyte/pull/17145) | Disable streams "export", "engage" on discover if not access | -| 0.1.24 | 2022-09-26 | [16915](https://github.com/airbytehq/airbyte/pull/16915) | Added Service Accounts support | -| 0.1.23 | 2022-09-18 | [16843](https://github.com/airbytehq/airbyte/pull/16843) | Add stream=True for `export` stream | -| 0.1.22 | 2022-09-15 | [16770](https://github.com/airbytehq/airbyte/pull/16770) | Use "Retry-After" header for backoff | -| 0.1.21 | 2022-09-11 | [16191](https://github.com/airbytehq/airbyte/pull/16191) | Improved connector's input configuration validation | -| 0.1.20 | 2022-08-22 | [15091](https://github.com/airbytehq/airbyte/pull/15091) | Improve `export` stream cursor support | -| 0.1.19 | 2022-08-18 | [15739](https://github.com/airbytehq/airbyte/pull/15739) | Update `titile` and `description` for `Project Secret` field | -| 0.1.18 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and specs | -| 0.1.17 | 2022-06-01 | [12801](https://github.com/airbytehq/airbyte/pull/13372) | Acceptance tests fix, fixing some bugs for beta release | -| 0.1.16 | 2022-05-30 | [12801](https://github.com/airbytehq/airbyte/pull/12801) | Add end_date parameter | -| 0.1.15 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | -| 0.1.14 | 2022-05-02 | [11501](https://github.com/airbytehq/airbyte/pull/11501) | Improve incremental sync method to streams | -| 0.1.13 | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | -| 0.1.12 | 2022-03-31 | [11633](https://github.com/airbytehq/airbyte/pull/11633) | Increase unit test coverage | -| 0.1.11 | 2022-04-04 | [11318](https://github.com/airbytehq/airbyte/pull/11318) | Change Response Reading | -| 0.1.10 | 2022-03-31 | [11227](https://github.com/airbytehq/airbyte/pull/11227) | Fix cohort id always null in the cohort_members stream | -| 0.1.9 | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | -| 0.1.7 | 2021-12-01 | [8381](https://github.com/airbytehq/airbyte/pull/8381) | Increased performance for `discovery` stage during connector setup | -| 0.1.6 | 2021-11-25 | [8256](https://github.com/airbytehq/airbyte/issues/8256) | Deleted `date_window_size` and fix schemas date type issue | -| 0.1.5 | 2021-11-10 | [7451](https://github.com/airbytehq/airbyte/issues/7451) | Support `start_date` older than 1 year | -| 0.1.4 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.3 | 2021-10-30 | [7505](https://github.com/airbytehq/airbyte/issues/7505) | Guarantee that standard and custom mixpanel properties in the `Engage` stream are written as strings | -| 0.1.2 | 2021-11-02 | [7439](https://github.com/airbytehq/airbyte/issues/7439) | Added delay for all streams to match API limitation of requests rate | -| 0.1.1 | 2021-09-16 | [6075](https://github.com/airbytehq/airbyte/issues/6075) | Added option to select project region | -| 0.1.0 | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------| +| 1.0.0 | 2023-09-27 | [30025](https://github.com/airbytehq/airbyte/pull/30025) | Fix type of datetime field in engage stream; fix primary key for export stream. | +| 0.1.41 | 2023-09-26 | [30149](https://github.com/airbytehq/airbyte/pull/30149) | Change config schema; set checkpointing interval; add suggested streams; add casting datetime fields. | +| 0.1.40 | 2022-09-20 | [30090](https://github.com/airbytehq/airbyte/pull/30090) | Handle 400 error when the credentials become expired | +| 0.1.39 | 2023-09-15 | [30469](https://github.com/airbytehq/airbyte/pull/30469) | Add default primary key `distinct_id` to `Export` stream | +| 0.1.38 | 2023-08-31 | [30028](https://github.com/airbytehq/airbyte/pull/30028) | Handle gracefully project timezone mismatch | +| 0.1.37 | 2023-07-20 | [27932](https://github.com/airbytehq/airbyte/pull/27932) | Fix spec: change start/end date format to `date` | +| 0.1.36 | 2023-06-27 | [27752](https://github.com/airbytehq/airbyte/pull/27752) | Partially revert version 0.1.32; Use exponential backoff; | +| 0.1.35 | 2023-06-12 | [27252](https://github.com/airbytehq/airbyte/pull/27252) | Add should_retry False for 402 error | +| 0.1.34 | 2023-05-15 | [21837](https://github.com/airbytehq/airbyte/pull/21837) | Add "insert_id" field to "export" stream schema | +| 0.1.33 | 2023-04-25 | [25543](https://github.com/airbytehq/airbyte/pull/25543) | Set should_retry for 104 error in stream export | +| 0.1.32 | 2023-04-11 | [25056](https://github.com/airbytehq/airbyte/pull/25056) | Set HttpAvailabilityStrategy, add exponential backoff, streams export and annotations add undeclared fields | +| 0.1.31 | 2023-02-13 | [22936](https://github.com/airbytehq/airbyte/pull/22936) | Specified date formatting in specification | +| 0.1.30 | 2023-01-27 | [22017](https://github.com/airbytehq/airbyte/pull/22017) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.29 | 2022-11-02 | [18846](https://github.com/airbytehq/airbyte/pull/18846) | For "export" stream make line parsing more robust | +| 0.1.28 | 2022-10-06 | [17699](https://github.com/airbytehq/airbyte/pull/17699) | Fix discover step issue cursor field None | +| 0.1.27 | 2022-09-29 | [17415](https://github.com/airbytehq/airbyte/pull/17415) | Disable stream "cohort_members" on discover if not access | +| 0.1.26 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | +| 0.1.25 | 2022-09-27 | [17145](https://github.com/airbytehq/airbyte/pull/17145) | Disable streams "export", "engage" on discover if not access | +| 0.1.24 | 2022-09-26 | [16915](https://github.com/airbytehq/airbyte/pull/16915) | Added Service Accounts support | +| 0.1.23 | 2022-09-18 | [16843](https://github.com/airbytehq/airbyte/pull/16843) | Add stream=True for `export` stream | +| 0.1.22 | 2022-09-15 | [16770](https://github.com/airbytehq/airbyte/pull/16770) | Use "Retry-After" header for backoff | +| 0.1.21 | 2022-09-11 | [16191](https://github.com/airbytehq/airbyte/pull/16191) | Improved connector's input configuration validation | +| 0.1.20 | 2022-08-22 | [15091](https://github.com/airbytehq/airbyte/pull/15091) | Improve `export` stream cursor support | +| 0.1.19 | 2022-08-18 | [15739](https://github.com/airbytehq/airbyte/pull/15739) | Update `titile` and `description` for `Project Secret` field | +| 0.1.18 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and specs | +| 0.1.17 | 2022-06-01 | [12801](https://github.com/airbytehq/airbyte/pull/13372) | Acceptance tests fix, fixing some bugs for beta release | +| 0.1.16 | 2022-05-30 | [12801](https://github.com/airbytehq/airbyte/pull/12801) | Add end_date parameter | +| 0.1.15 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | +| 0.1.14 | 2022-05-02 | [11501](https://github.com/airbytehq/airbyte/pull/11501) | Improve incremental sync method to streams | +| 0.1.13 | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | +| 0.1.12 | 2022-03-31 | [11633](https://github.com/airbytehq/airbyte/pull/11633) | Increase unit test coverage | +| 0.1.11 | 2022-04-04 | [11318](https://github.com/airbytehq/airbyte/pull/11318) | Change Response Reading | +| 0.1.10 | 2022-03-31 | [11227](https://github.com/airbytehq/airbyte/pull/11227) | Fix cohort id always null in the cohort_members stream | +| 0.1.9 | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | +| 0.1.7 | 2021-12-01 | [8381](https://github.com/airbytehq/airbyte/pull/8381) | Increased performance for `discovery` stage during connector setup | +| 0.1.6 | 2021-11-25 | [8256](https://github.com/airbytehq/airbyte/issues/8256) | Deleted `date_window_size` and fix schemas date type issue | +| 0.1.5 | 2021-11-10 | [7451](https://github.com/airbytehq/airbyte/issues/7451) | Support `start_date` older than 1 year | +| 0.1.4 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.3 | 2021-10-30 | [7505](https://github.com/airbytehq/airbyte/issues/7505) | Guarantee that standard and custom mixpanel properties in the `Engage` stream are written as strings | +| 0.1.2 | 2021-11-02 | [7439](https://github.com/airbytehq/airbyte/issues/7439) | Added delay for all streams to match API limitation of requests rate | +| 0.1.1 | 2021-09-16 | [6075](https://github.com/airbytehq/airbyte/issues/6075) | Added option to select project region | +| 0.1.0 | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector | From aee4755dea0c99778c9fb67470bcd20ce5a7a32e Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 28 Sep 2023 11:45:29 +0300 Subject: [PATCH 4/8] Add comment for custom testing environment --- .../connectors/source-mixpanel/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml index 412e74f7642df..9812edcc6143f 100644 --- a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-mixpanel:dev +# custom configuration is used for tests to speed up testing and avoid hitting rate limits custom_environment_variables: REQS_PER_HOUR_LIMIT: 0 AVAILABLE_TESTING_RANGE_DAYS: 10 From b3d22a92e75e16b2e336d2b9f6acd6202a3a26fc Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Fri, 29 Sep 2023 18:31:29 -0400 Subject: [PATCH 5/8] clarify changes need to happen after applying the upgrade --- airbyte-integrations/connectors/source-mixpanel/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index f537d8018b70e..e139fa3b6d0f1 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -21,7 +21,7 @@ data: releases: breakingChanges: 1.0.0: - message: In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data before updating. + message: In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data after upgrading. upgradeDeadline: "2023-10-31" suggestedStreams: streams: From e87e3c47a6dff0ef354f2bc91ce659fd000434d5 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Fri, 29 Sep 2023 18:31:53 -0400 Subject: [PATCH 6/8] clarify changes need to happen after applying the upgrade --- docs/integrations/sources/mixpanel-migrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/mixpanel-migrations.md b/docs/integrations/sources/mixpanel-migrations.md index 7d2a76dcca0d9..ccf2bf4294a87 100644 --- a/docs/integrations/sources/mixpanel-migrations.md +++ b/docs/integrations/sources/mixpanel-migrations.md @@ -2,4 +2,4 @@ ## Upgrading to 1.0.0 -In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data before updating. \ No newline at end of file +In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data after upgrading. From 56ff05661c6b261b0921da7032d01400f95c4e1c Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Fri, 29 Sep 2023 19:39:40 -0400 Subject: [PATCH 7/8] messaging tweaks --- airbyte-integrations/connectors/source-mixpanel/metadata.yaml | 2 +- docs/integrations/sources/mixpanel-migrations.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index e139fa3b6d0f1..c48242555516b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -21,7 +21,7 @@ data: releases: breakingChanges: 1.0.0: - message: In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data after upgrading. + message: In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh the source schema and reset affected streams after upgrading. upgradeDeadline: "2023-10-31" suggestedStreams: streams: diff --git a/docs/integrations/sources/mixpanel-migrations.md b/docs/integrations/sources/mixpanel-migrations.md index ccf2bf4294a87..316406819e712 100644 --- a/docs/integrations/sources/mixpanel-migrations.md +++ b/docs/integrations/sources/mixpanel-migrations.md @@ -2,4 +2,4 @@ ## Upgrading to 1.0.0 -In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh their schemas and data after upgrading. +In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh the source schema and reset affected streams after upgrading. From 5068013a85d97ec3e1d19fe12c8dd9802198d9d4 Mon Sep 17 00:00:00 2001 From: pedroslopez Date: Mon, 2 Oct 2023 17:01:41 +0000 Subject: [PATCH 8/8] Automated Commit - Formatting Changes --- .../connectors/source-mixpanel/unit_tests/test_streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index 8645cc6d62d4f..de54d1c6b89fc 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -284,7 +284,7 @@ def engage_schema_response(): "$CreatedDateTimestamp": {"count": 300, "type": "number"}, "$CreatedDate": {"count": 11, "type": "datetime"}, "$properties": {"count": 2, "type": "object"}, - "$tags": {"count": 131, "type": "list"} + "$tags": {"count": 131, "type": "list"}, } }, ) @@ -316,9 +316,9 @@ def test_engage_schema(requests_mock, engage_schema_response, config): "region": {"type": ["null", "string"]}, "tags": {"items": {}, "required": False, "type": ["null", "array"]}, "timezone": {"type": ["null", "string"]}, - "unblocked": {"type": ["null", "string"]} + "unblocked": {"type": ["null", "string"]}, }, - "type": "object" + "type": "object", }