From 78728ff6b56cf0eeb4f78b5410a9e875038a9f30 Mon Sep 17 00:00:00 2001 From: Augustin Date: Thu, 29 Sep 2022 21:03:31 +0200 Subject: [PATCH] source-bing-ads: migrate to per-stream state (#17386) --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-bing-ads/Dockerfile | 2 +- .../integration_tests/state.json | 52 ++++++++++++++----- .../source_bing_ads/reports.py | 2 +- .../unit_tests/test_reports.py | 14 ++--- docs/integrations/sources/bing-ads.md | 1 + 7 files changed, 50 insertions(+), 25 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 13b28a71f2e73..bc3e0f56e335b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -128,7 +128,7 @@ - name: Bing Ads sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 dockerRepository: airbyte/source-bing-ads - dockerImageTag: 0.1.12 + dockerImageTag: 0.1.13 documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads icon: bingads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 4b29a1df4e35a..aa404f885a6e8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1460,7 +1460,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/source-bing-ads:0.1.12" +- dockerImage: "airbyte/source-bing-ads:0.1.13" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index 6e35e1ee00560..d06520aef7d58 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json index 26f12335c9e30..ef99548a8bb29 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json @@ -1,22 +1,46 @@ -{ - "keyword_performance_report_hourly": { - "180278106": { - "TimePeriod": 1627820152 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": 1627820152 + } + }, + "stream_descriptor": { "name": "keyword_performance_report_hourly" } } }, - "budget_summary_report_hourly": { - "180278106": { - "Date": 1627800152 + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "Date": 1627800152 + } + }, + "stream_descriptor": { "name": "budget_summary_report_hourly" } } }, - "ad_performance_report_hourly": { - "180278106": { - "TimePeriod": 1627795152 + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": 1627795152 + } + }, + "stream_descriptor": { "name": "ad_performance_report_hourly" } } }, - "campaign_performance_report_hourly": { - "180278106": { - "TimePeriod": 1727810152 + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": 1727810152 + } + }, + "stream_descriptor": { "name": "campaign_performance_report_hourly" } } } -} +] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py index f62092e7d8f1f..c5420e1537862 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py @@ -233,7 +233,7 @@ def get_updated_state( current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any], ) -> Mapping[str, Any]: - account_id = latest_record["AccountId"] + account_id = str(latest_record["AccountId"]) current_stream_state[account_id] = current_stream_state.get(account_id, {}) current_stream_state[account_id][self.cursor_field] = max( self.get_report_record_timestamp(latest_record[self.cursor_field]), diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 5ec08a203f585..491dfd8f01a59 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -40,15 +40,15 @@ def test_get_column_value(): def test_get_updated_state_new_state(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-01").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-01").timestamp()}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert new_state[123]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert new_state["123"]["Time"] == pendulum.parse("2020-01-02").timestamp() -def test_get_updated_state_state_uncahanged(): +def test_get_updated_state_state_unchanged(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(copy.deepcopy(stream_state), latest_record) assert stream_state == new_state @@ -56,11 +56,11 @@ def test_get_updated_state_state_uncahanged(): def test_get_updated_state_state_new_account(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} latest_record = {"AccountId": 234, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert 234 in new_state and 123 in new_state - assert new_state[234]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert "234" in new_state and "123" in new_state + assert new_state["234"]["Time"] == pendulum.parse("2020-01-02").timestamp() def test_get_report_record_timestamp_daily(): diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 9bfbf1736d8c3..ee38395c8cff7 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -96,6 +96,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | +| 0.1.13 | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386) | Migrate to per-stream states. | | 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | | 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable | | 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set |