From bb13081f6f0871c0d425bbfd0247d90c38a97f3a Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 16 Oct 2023 10:20:40 +0300 Subject: [PATCH 1/6] Source Bing Ads: remove primary key for the geographic performance reports --- .../connectors/source-bing-ads/Dockerfile | 2 +- .../connectors/source-bing-ads/metadata.yaml | 3 ++ .../source-bing-ads/source_bing_ads/source.py | 4 +++ .../unit_tests/test_reports.py | 30 ++++++++++++++++++- .../sources/bing-ads-migrations.md | 10 +++++++ docs/integrations/sources/bing-ads.md | 1 + 6 files changed, 48 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index 079d1b57db78e..e78ff4576dc04 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 79a9adc06b4ab..acf4a3ef005c9 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -34,5 +34,8 @@ data: 1.0.0: message: "Version 1.0.0 removes the primary keys from the geographic performance report streams. This will prevent the connector from losing data in the incremental append+dedup sync mode because of deduplication and incorrect primary keys. A data reset and schema refresh of all the affected streams is required for the changes to take effect." upgradeDeadline: "2023-10-25" + 2.0.0: + message: "This update is a follow-up to the previous major update - version 1.0.0. It completes what should have been done in version 1.0.0, but unfortunately was not. Version 2.0.0 removes the primary keys from the geographic performance report streams. This will prevent the connector from losing data in the incremental append+dedup sync mode because of deduplication and incorrect primary keys. A data reset and schema refresh of all the affected streams is required for the changes to take effect." + upgradeDeadline: "2023-10-31" supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index da126a55b1b56..7d32cb18b6cef 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -699,6 +699,10 @@ class GeographicPerformanceReport(PerformanceReportsMixin, BingAdsStream): cursor_field = "TimePeriod" report_schema_name = "geographic_performance_report" + # Need to override the primary key here because the one inherited from the PerformanceReportsMixin + # is incorrect for the geographic performance reports + primary_key = None + report_columns = [ "AccountId", "CampaignId", diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 4d82a559d91f6..a540df2e1c646 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -3,11 +3,19 @@ # import copy +from unittest.mock import Mock import pendulum +import pytest from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord, _RowValues from source_bing_ads.reports import PerformanceReportsMixin, ReportsMixin -from source_bing_ads.source import SourceBingAds +from source_bing_ads.source import ( + GeographicPerformanceReportDaily, + GeographicPerformanceReportHourly, + GeographicPerformanceReportMonthly, + GeographicPerformanceReportWeekly, + SourceBingAds, +) class TestClient: @@ -137,3 +145,23 @@ def test_report_get_start_date_performance_report_wo_stream_state(): stream_state = {} account_id = "123" assert reports_start_date.subtract(days=days_to_subtract) == test_report.get_start_date(stream_state, account_id) + + +@pytest.mark.parametrize( + "performance_report_cls", + ( + GeographicPerformanceReportDaily, + GeographicPerformanceReportHourly, + GeographicPerformanceReportMonthly, + GeographicPerformanceReportWeekly + ) +) +def test_geographic_performance_report_pk(performance_report_cls): + config = { + "developer_token": "developer_token", + "client_id": "client_id", + "refresh_token": "refresh_token", + "reports_start_date": "2020-01-01T00:00:00Z" + } + stream = performance_report_cls(client=Mock(), config=config) + assert stream.primary_key is None diff --git a/docs/integrations/sources/bing-ads-migrations.md b/docs/integrations/sources/bing-ads-migrations.md index 3d378d2517adb..04b59e7ac319f 100644 --- a/docs/integrations/sources/bing-ads-migrations.md +++ b/docs/integrations/sources/bing-ads-migrations.md @@ -7,4 +7,14 @@ This version update only affects the geographic performance reports streams. Version 1.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. Due to multiple records with the same primary key, users could experience data loss in the incremental append+dedup mode because of deduplication. +For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. + +## Upgrading to 2.0.0 + +This update is a follow-up to the previous major update - version 1.0.0. It completes what should have been done in version 1.0.0, but unfortunately was not. +It only affects the geographic performance reports streams. + +Version 2.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. +Due to multiple records with the same primary key, users could experience data loss in the incremental append+dedup mode because of deduplication. + For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. \ No newline at end of file diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index f5d8bbc64b880..ab77bd648ce75 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -123,6 +123,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.0 | 2023-10-16 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | | 1.0.0 | 2023-10-11 | [31277](https://github.com/airbytehq/airbyte/pull/31277) | Remove primary keys from the geographic performance reports. | | 0.2.3 | 2023-09-28 | [30834](https://github.com/airbytehq/airbyte/pull/30834) | Wrap auth error with the config error. | | 0.2.2 | 2023-09-27 | [30791](https://github.com/airbytehq/airbyte/pull/30791) | Fix missing fields for geographic performance reports. | From df6feb59317a28e1d036f76b5b1e635d12ab9681 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 16 Oct 2023 10:27:23 +0300 Subject: [PATCH 2/6] update changelog --- docs/integrations/sources/bing-ads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index ab77bd648ce75..c26dc6062ccb3 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -123,7 +123,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------| -| 2.0.0 | 2023-10-16 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | +| 2.0.0 | 2023-10-16 | [31432](https://github.com/airbytehq/airbyte/pull/31432) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | | 1.0.0 | 2023-10-11 | [31277](https://github.com/airbytehq/airbyte/pull/31277) | Remove primary keys from the geographic performance reports. | | 0.2.3 | 2023-09-28 | [30834](https://github.com/airbytehq/airbyte/pull/30834) | Wrap auth error with the config error. | | 0.2.2 | 2023-09-27 | [30791](https://github.com/airbytehq/airbyte/pull/30791) | Fix missing fields for geographic performance reports. | From 2fd11d2594a7d524edc3b6540cfac0f8f3b36f95 Mon Sep 17 00:00:00 2001 From: davydov-d Date: Mon, 16 Oct 2023 07:50:25 +0000 Subject: [PATCH 3/6] Automated Commit - Formatting Changes --- .../connectors/source-bing-ads/unit_tests/test_reports.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index a540df2e1c646..8763c1ea8bcf0 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -153,15 +153,15 @@ def test_report_get_start_date_performance_report_wo_stream_state(): GeographicPerformanceReportDaily, GeographicPerformanceReportHourly, GeographicPerformanceReportMonthly, - GeographicPerformanceReportWeekly - ) + GeographicPerformanceReportWeekly, + ), ) def test_geographic_performance_report_pk(performance_report_cls): config = { "developer_token": "developer_token", "client_id": "client_id", "refresh_token": "refresh_token", - "reports_start_date": "2020-01-01T00:00:00Z" + "reports_start_date": "2020-01-01T00:00:00Z", } stream = performance_report_cls(client=Mock(), config=config) assert stream.primary_key is None From e24202bf95bbd5829538ba4b264e351a7b3bc60e Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 16 Oct 2023 11:49:07 +0300 Subject: [PATCH 4/6] increment connector version --- airbyte-integrations/connectors/source-bing-ads/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index acf4a3ef005c9..0e1b54fc9ddb4 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 1.0.0 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-bing-ads githubIssueLabel: source-bing-ads icon: bingads.svg From 88aa84fdcffa12d2087fd806d6b2a07a3410116d Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 16 Oct 2023 12:29:37 +0300 Subject: [PATCH 5/6] update migration order in the docs --- docs/integrations/sources/bing-ads-migrations.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/integrations/sources/bing-ads-migrations.md b/docs/integrations/sources/bing-ads-migrations.md index 04b59e7ac319f..91a0761cb6a34 100644 --- a/docs/integrations/sources/bing-ads-migrations.md +++ b/docs/integrations/sources/bing-ads-migrations.md @@ -1,20 +1,20 @@ # Bing Ads Migration Guide -## Upgrading to 1.0.0 +## Upgrading to 2.0.0 -This version update only affects the geographic performance reports streams. +This update is a follow-up to the previous major update - version 1.0.0. It completes what should have been done in version 1.0.0, but unfortunately was not. +It only affects the geographic performance reports streams. -Version 1.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. +Version 2.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. Due to multiple records with the same primary key, users could experience data loss in the incremental append+dedup mode because of deduplication. For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. -## Upgrading to 2.0.0 +## Upgrading to 1.0.0 -This update is a follow-up to the previous major update - version 1.0.0. It completes what should have been done in version 1.0.0, but unfortunately was not. -It only affects the geographic performance reports streams. +This version update only affects the geographic performance reports streams. -Version 2.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. +Version 1.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. Due to multiple records with the same primary key, users could experience data loss in the incremental append+dedup mode because of deduplication. -For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. \ No newline at end of file +For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. From 885c0ba8748cf5f92497454f1b97d10376421da6 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 17 Oct 2023 12:41:53 +0300 Subject: [PATCH 6/6] change major version bump to patch --- .../connectors/source-bing-ads/Dockerfile | 2 +- .../connectors/source-bing-ads/metadata.yaml | 5 +---- docs/integrations/sources/bing-ads-migrations.md | 12 +----------- docs/integrations/sources/bing-ads.md | 2 +- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index e78ff4576dc04..5cf61c8a10678 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 0e1b54fc9ddb4..bbba8cb69f28f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 2.0.0 + dockerImageTag: 1.0.1 dockerRepository: airbyte/source-bing-ads githubIssueLabel: source-bing-ads icon: bingads.svg @@ -34,8 +34,5 @@ data: 1.0.0: message: "Version 1.0.0 removes the primary keys from the geographic performance report streams. This will prevent the connector from losing data in the incremental append+dedup sync mode because of deduplication and incorrect primary keys. A data reset and schema refresh of all the affected streams is required for the changes to take effect." upgradeDeadline: "2023-10-25" - 2.0.0: - message: "This update is a follow-up to the previous major update - version 1.0.0. It completes what should have been done in version 1.0.0, but unfortunately was not. Version 2.0.0 removes the primary keys from the geographic performance report streams. This will prevent the connector from losing data in the incremental append+dedup sync mode because of deduplication and incorrect primary keys. A data reset and schema refresh of all the affected streams is required for the changes to take effect." - upgradeDeadline: "2023-10-31" supportLevel: certified metadataSpecVersion: "1.0" diff --git a/docs/integrations/sources/bing-ads-migrations.md b/docs/integrations/sources/bing-ads-migrations.md index 91a0761cb6a34..3d378d2517adb 100644 --- a/docs/integrations/sources/bing-ads-migrations.md +++ b/docs/integrations/sources/bing-ads-migrations.md @@ -1,15 +1,5 @@ # Bing Ads Migration Guide -## Upgrading to 2.0.0 - -This update is a follow-up to the previous major update - version 1.0.0. It completes what should have been done in version 1.0.0, but unfortunately was not. -It only affects the geographic performance reports streams. - -Version 2.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. -Due to multiple records with the same primary key, users could experience data loss in the incremental append+dedup mode because of deduplication. - -For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. - ## Upgrading to 1.0.0 This version update only affects the geographic performance reports streams. @@ -17,4 +7,4 @@ This version update only affects the geographic performance reports streams. Version 1.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. Due to multiple records with the same primary key, users could experience data loss in the incremental append+dedup mode because of deduplication. -For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. +For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. \ No newline at end of file diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index c26dc6062ccb3..c967c348bcece 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -123,7 +123,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------| -| 2.0.0 | 2023-10-16 | [31432](https://github.com/airbytehq/airbyte/pull/31432) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | +| 1.0.1 | 2023-10-16 | [31432](https://github.com/airbytehq/airbyte/pull/31432) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | | 1.0.0 | 2023-10-11 | [31277](https://github.com/airbytehq/airbyte/pull/31277) | Remove primary keys from the geographic performance reports. | | 0.2.3 | 2023-09-28 | [30834](https://github.com/airbytehq/airbyte/pull/30834) | Wrap auth error with the config error. | | 0.2.2 | 2023-09-27 | [30791](https://github.com/airbytehq/airbyte/pull/30791) | Fix missing fields for geographic performance reports. |