From 1c72e57a31019acff85d7f19013a2cd6756eb276 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 29 Sep 2022 14:49:28 +0200 Subject: [PATCH 1/5] migrate --- .../connectors/source-bing-ads/Dockerfile | 2 +- .../integration_tests/state.json | 36 +++++++++++-------- .../source_bing_ads/reports.py | 2 +- docs/integrations/sources/bing-ads.md | 31 ++++++++-------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index 6e35e1ee00560..d06520aef7d58 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json index 26f12335c9e30..2732c19d48491 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json @@ -1,22 +1,30 @@ -{ - "keyword_performance_report_hourly": { - "180278106": { - "TimePeriod": 1627820152 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "180278106": { "TimePeriod": 1627820152 } }, + "stream_descriptor": { "name": "keyword_performance_report_hourly" } } }, - "budget_summary_report_hourly": { - "180278106": { - "Date": 1627800152 + { + "type": "STREAM", + "stream": { + "stream_state": { "180278106": { "Date": 1627800152 } }, + "stream_descriptor": { "name": "budget_summary_report_hourly" } } }, - "ad_performance_report_hourly": { - "180278106": { - "TimePeriod": 1627795152 + { + "type": "STREAM", + "stream": { + "stream_state": { "180278106": { "TimePeriod": 1627795152 } }, + "stream_descriptor": { "name": "ad_performance_report_hourly" } } }, - "campaign_performance_report_hourly": { - "180278106": { - "TimePeriod": 1727810152 + { + "type": "STREAM", + "stream": { + "stream_state": { "180278106": { "TimePeriod": 1727810152 } }, + "stream_descriptor": { "name": "campaign_performance_report_hourly" } } } -} +] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py index f62092e7d8f1f..c5420e1537862 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py @@ -233,7 +233,7 @@ def get_updated_state( current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any], ) -> Mapping[str, Any]: - account_id = latest_record["AccountId"] + account_id = str(latest_record["AccountId"]) current_stream_state[account_id] = current_stream_state.get(account_id, {}) current_stream_state[account_id][self.cursor_field] = max( self.get_report_record_timestamp(latest_record[self.cursor_field]), diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 1a87e7b408dd9..a5608ad243800 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -94,18 +94,19 @@ A report's aggregation window is indicated in its name. For example, `account_pe The Bing Ads API limits the number of requests for all Microsoft Advertising clients. You can find detailed info [here](https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling). ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | -| 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | -| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987))| Fixed log messages being unreadable | -| 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | -| 0.1.9 | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862) | Added missing columns | -| 0.1.8 | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801) | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration | -| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | -| 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | -| 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | -| 0.1.3 | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510) | Fixed broken dependency that blocked connector's operations | -| 0.1.2 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | -| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | +| 0.1.13 | 2022-09-29 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Migrate to per-stream states. | +| 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | +| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable | +| 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | +| 0.1.9 | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862) | Added missing columns | +| 0.1.8 | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801) | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration | +| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | +| 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | +| 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | +| 0.1.3 | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510) | Fixed broken dependency that blocked connector's operations | +| 0.1.2 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | +| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | From a3775fb3aeca3588b4aee9b79c9dca83305506c6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 29 Sep 2022 14:53:28 +0200 Subject: [PATCH 2/5] migrate --- docs/integrations/sources/bing-ads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index a5608ad243800..ee38395c8cff7 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -96,7 +96,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | -| 0.1.13 | 2022-09-29 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Migrate to per-stream states. | +| 0.1.13 | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386) | Migrate to per-stream states. | | 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | | 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable | | 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | From 1f72dd81fb3b75cd2868996f54b02254a0418ae6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 29 Sep 2022 16:40:50 +0200 Subject: [PATCH 3/5] update unit tests --- .../source-bing-ads/unit_tests/test_reports.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 5ec08a203f585..491dfd8f01a59 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -40,15 +40,15 @@ def test_get_column_value(): def test_get_updated_state_new_state(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-01").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-01").timestamp()}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert new_state[123]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert new_state["123"]["Time"] == pendulum.parse("2020-01-02").timestamp() -def test_get_updated_state_state_uncahanged(): +def test_get_updated_state_state_unchanged(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(copy.deepcopy(stream_state), latest_record) assert stream_state == new_state @@ -56,11 +56,11 @@ def test_get_updated_state_state_uncahanged(): def test_get_updated_state_state_new_account(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} latest_record = {"AccountId": 234, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert 234 in new_state and 123 in new_state - assert new_state[234]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert "234" in new_state and "123" in new_state + assert new_state["234"]["Time"] == pendulum.parse("2020-01-02").timestamp() def test_get_report_record_timestamp_daily(): From b37fad148c800733c7c45be2abc66aaac154a1aa Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 29 Sep 2022 20:43:27 +0200 Subject: [PATCH 4/5] reformat state.json --- .../integration_tests/state.json | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json index 2732c19d48491..ef99548a8bb29 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json @@ -2,28 +2,44 @@ { "type": "STREAM", "stream": { - "stream_state": { "180278106": { "TimePeriod": 1627820152 } }, + "stream_state": { + "180278106": { + "TimePeriod": 1627820152 + } + }, "stream_descriptor": { "name": "keyword_performance_report_hourly" } } }, { "type": "STREAM", "stream": { - "stream_state": { "180278106": { "Date": 1627800152 } }, + "stream_state": { + "180278106": { + "Date": 1627800152 + } + }, "stream_descriptor": { "name": "budget_summary_report_hourly" } } }, { "type": "STREAM", "stream": { - "stream_state": { "180278106": { "TimePeriod": 1627795152 } }, + "stream_state": { + "180278106": { + "TimePeriod": 1627795152 + } + }, "stream_descriptor": { "name": "ad_performance_report_hourly" } } }, { "type": "STREAM", "stream": { - "stream_state": { "180278106": { "TimePeriod": 1727810152 } }, + "stream_state": { + "180278106": { + "TimePeriod": 1727810152 + } + }, "stream_descriptor": { "name": "campaign_performance_report_hourly" } } } From 6b7e5cb39f4304b291c6d9db0ab29efdf07a4641 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 29 Sep 2022 18:56:43 +0000 Subject: [PATCH 5/5] auto-bump connector version [ci skip] --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f11ff7818dd5c..3a36363ff29c0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -128,7 +128,7 @@ - name: Bing Ads sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 dockerRepository: airbyte/source-bing-ads - dockerImageTag: 0.1.12 + dockerImageTag: 0.1.13 documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads icon: bingads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5f579e691d2a9..e379f74e1f132 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1460,7 +1460,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/source-bing-ads:0.1.12" +- dockerImage: "airbyte/source-bing-ads:0.1.13" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads" connectionSpecification: