source-bing-ads: migrate to per-stream state (#17386)

airbytehq · Sep 29, 2022 · 99955ce · 99955ce
1 parent 33bad16
commit 99955ce
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 25 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -128,7 +128,7 @@
 - name: Bing Ads
   sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331
   dockerRepository: airbyte/source-bing-ads
-  dockerImageTag: 0.1.12
+  dockerImageTag: 0.1.13
   documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads
   icon: bingads.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1460,7 +1460,7 @@
     - "overwrite"
     - "append"
     - "append_dedup"
-- dockerImage: "airbyte/source-bing-ads:0.1.12"
+- dockerImage: "airbyte/source-bing-ads:0.1.13"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=0.1.13
 LABEL io.airbyte.name=airbyte/source-bing-ads
diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json
@@ -1,22 +1,46 @@
-{
-  "keyword_performance_report_hourly": {
-    "180278106": {
-      "TimePeriod": 1627820152
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "180278106": {
+          "TimePeriod": 1627820152
+        }
+      },
+      "stream_descriptor": { "name": "keyword_performance_report_hourly" }
     }
   },
-  "budget_summary_report_hourly": {
-    "180278106": {
-      "Date": 1627800152
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "180278106": {
+          "Date": 1627800152
+        }
+      },
+      "stream_descriptor": { "name": "budget_summary_report_hourly" }
     }
   },
-  "ad_performance_report_hourly": {
-    "180278106": {
-      "TimePeriod": 1627795152
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "180278106": {
+          "TimePeriod": 1627795152
+        }
+      },
+      "stream_descriptor": { "name": "ad_performance_report_hourly" }
     }
   },
-  "campaign_performance_report_hourly": {
-    "180278106": {
-      "TimePeriod": 1727810152
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "180278106": {
+          "TimePeriod": 1727810152
+        }
+      },
+      "stream_descriptor": { "name": "campaign_performance_report_hourly" }
     }
   }
-}
+]
diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py
@@ -233,7 +233,7 @@ def get_updated_state(
         current_stream_state: MutableMapping[str, Any],
         latest_record: Mapping[str, Any],
     ) -> Mapping[str, Any]:
-        account_id = latest_record["AccountId"]
+        account_id = str(latest_record["AccountId"])
         current_stream_state[account_id] = current_stream_state.get(account_id, {})
         current_stream_state[account_id][self.cursor_field] = max(
             self.get_report_record_timestamp(latest_record[self.cursor_field]),

diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py
@@ -40,27 +40,27 @@ def test_get_column_value():
 
 def test_get_updated_state_new_state():
     test_report = TestReport()
-    stream_state = {123: {"Time": pendulum.parse("2020-01-01").timestamp()}}
+    stream_state = {"123": {"Time": pendulum.parse("2020-01-01").timestamp()}}
     latest_record = {"AccountId": 123, "Time": "2020-01-02"}
     new_state = test_report.get_updated_state(stream_state, latest_record)
-    assert new_state[123]["Time"] == pendulum.parse("2020-01-02").timestamp()
+    assert new_state["123"]["Time"] == pendulum.parse("2020-01-02").timestamp()
 
 
-def test_get_updated_state_state_uncahanged():
+def test_get_updated_state_state_unchanged():
     test_report = TestReport()
-    stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}}
+    stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}}
     latest_record = {"AccountId": 123, "Time": "2020-01-02"}
     new_state = test_report.get_updated_state(copy.deepcopy(stream_state), latest_record)
     assert stream_state == new_state
 
 
 def test_get_updated_state_state_new_account():
     test_report = TestReport()
-    stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}}
+    stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}}
     latest_record = {"AccountId": 234, "Time": "2020-01-02"}
     new_state = test_report.get_updated_state(stream_state, latest_record)
-    assert 234 in new_state and 123 in new_state
-    assert new_state[234]["Time"] == pendulum.parse("2020-01-02").timestamp()
+    assert "234" in new_state and "123" in new_state
+    assert new_state["234"]["Time"] == pendulum.parse("2020-01-02").timestamp()
 
 
 def test_get_report_record_timestamp_daily():

diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md
@@ -96,6 +96,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli
 ## Changelog
 | Version | Date       | Pull Request                                                                                                                     | Subject                                                                                                                    |
 | :------ | :--------- | :------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- |
+| 0.1.13  | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386)                                                                         | Migrate to per-stream states.                                                                                              |
 | 0.1.12  | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335)                                                                         | Added backoff for socket.timeout                                                                                           |
 | 0.1.11  | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable                                                                                        |
 | 0.1.10  | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602)                                                                         | Fixed bug caused Hourly Reports to crash due to invalid fields set                                                         |