#259 on call: Source Hubspot - fix for the property_history stream wh…

…ich did not emit any records

airbyte-integrations/connectors/source-hubspot/Dockerfile

@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.66
+LABEL io.airbyte.version=0.1.67
 LABEL io.airbyte.name=airbyte/source-hubspot

airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml

@@ -18,88 +18,32 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       timeout_seconds: 600
-      configured_catalog_path: "sample_files/full_refresh_catalog.json"
-      empty_streams: ["workflows", "form_submissions", "ticket_pipelines", "property_history"]
+      configured_catalog_path: "sample_files/basic_read_catalog.json"
+      empty_streams: ["workflows", "form_submissions", "ticket_pipelines"]
+      expect_records:
+        path: "integration_tests/expected_records.txt"
     - config_path: "secrets/config_oauth.json"
       timeout_seconds: 600
-      configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json"
-      # The `campaigns` stream is empty in this case, because we use a catalog with
-      # incremental streams: subscription_changes and email_events (it takes a long time to read)
-      # and therefore the start date is set at 2021-10-10 for `config_oauth.json`,
-      # but the campaign was created on 2021-01-11
-      empty_streams: ["campaigns", "workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines", "property_history"]
+      configured_catalog_path: "sample_files/basic_read_oauth_catalog.json"
+      empty_streams: ["workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines"]
+      expect_records:
+        path: "integration_tests/expected_records.txt"
   incremental:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "sample_files/configured_catalog.json"
+      configured_catalog_path: "sample_files/incremental_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
+      # This config is the same as for basic read except for missing `property_history` stream.
+      # This stream emits records with dynamic timestamps under some conditions
+      # (properties like `hs_time_in_lead`, `hs_time_in_subscriber` etc.) which makes it impossible to compare records.
+      # Ignoring these fields makes testing this stream senseless because those valuable values of other property types
+      # will also be ignored. Instead, we test this stream against expected records in basic read test.
       configured_catalog_path: "sample_files/full_refresh_catalog.json"
-      ignored_fields:
-        "companies": ["properties", "hs_time_in_customer"]
-        "companies": ["properties", "hs_time_in_evangelist"]
-        "companies": ["properties", "hs_time_in_lead"]
-        "companies": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "companies": ["properties", "hs_time_in_opportunity"]
-        "companies": ["properties", "hs_time_in_other"]
-        "companies": ["properties", "hs_time_in_salesqualifiedlead"]
-        "companies": ["properties", "hs_time_in_subscriber"]
-        "contacts": ["properties", "hs_time_in_customer"]
-        "contacts": ["properties", "hs_time_in_evangelist"]
-        "contacts": ["properties", "hs_time_in_lead"]
-        "contacts": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_opportunity"]
-        "contacts": ["properties", "hs_time_in_other"]
-        "contacts": ["properties", "hs_time_in_salesqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_subscriber"]
-        "deals": ["properties", "hs_time_in_9567448"]
-        "deals": ["properties", "hs_time_in_9567449"]
-        "deals": ["properties", "hs_time_in_appointmentscheduled"]
-        "deals": ["properties", "hs_time_in_closedlost"]
-        "deals": ["properties", "hs_time_in_closedwon"]
-        "deals": ["properties", "hs_time_in_contractsent"]
-        "deals": ["properties", "hs_time_in_customclosedwonstage"]
-        "deals": ["properties", "hs_time_in_decisionmakerboughtin"]
-        "deals": ["properties", "hs_time_in_presentationscheduled"]
-        "deals": ["properties", "hs_time_in_qualifiedtobuy"]
-        "tickets": ["properties", "hs_time_in_1"]
-        "tickets": ["properties", "hs_time_in_2"]
-        "tickets": ["properties", "hs_time_in_3"]
-        "tickets": ["properties", "hs_time_in_4"]
-        "property_history": ["property", "hs_time_in_lead"]
-        "property_history": ["property", "hs_time_in_subscriber"]
     - config_path: "secrets/config_oauth.json"
-      configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json"
-      ignored_fields:
-        "companies": ["properties", "hs_time_in_customer"]
-        "companies": ["properties", "hs_time_in_evangelist"]
-        "companies": ["properties", "hs_time_in_lead"]
-        "companies": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "companies": ["properties", "hs_time_in_opportunity"]
-        "companies": ["properties", "hs_time_in_other"]
-        "companies": ["properties", "hs_time_in_salesqualifiedlead"]
-        "companies": ["properties", "hs_time_in_subscriber"]
-        "contacts": ["properties", "hs_time_in_customer"]
-        "contacts": ["properties", "hs_time_in_evangelist"]
-        "contacts": ["properties", "hs_time_in_lead"]
-        "contacts": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_opportunity"]
-        "contacts": ["properties", "hs_time_in_other"]
-        "contacts": ["properties", "hs_time_in_salesqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_subscriber"]
-        "deals": ["properties", "hs_time_in_9567448"]
-        "deals": ["properties", "hs_time_in_9567449"]
-        "deals": ["properties", "hs_time_in_appointmentscheduled"]
-        "deals": ["properties", "hs_time_in_closedlost"]
-        "deals": ["properties", "hs_time_in_closedwon"]
-        "deals": ["properties", "hs_time_in_contractsent"]
-        "deals": ["properties", "hs_time_in_customclosedwonstage"]
-        "deals": ["properties", "hs_time_in_decisionmakerboughtin"]
-        "deals": ["properties", "hs_time_in_presentationscheduled"]
-        "deals": ["properties", "hs_time_in_qualifiedtobuy"]
-        "tickets": ["properties", "hs_time_in_1"]
-        "tickets": ["properties", "hs_time_in_2"]
-        "tickets": ["properties", "hs_time_in_3"]
-        "tickets": ["properties", "hs_time_in_4"]
-        "property_history": ["property", "hs_time_in_lead"]
-        "property_history": ["property", "hs_time_in_subscriber"]
+      # This config is the same as for basic read except for missing `property_history` stream.
+      # This stream emits records with dynamic timestamps under some conditions
+      # (properties like `hs_time_in_lead`, `hs_time_in_subscriber` etc.) which makes it impossible to compare records.
+      # Ignoring these fields makes testing this stream senseless because those valuable values of other property types
+      # will also be ignored. Instead, we test this stream against expected records in basic read test.
+      configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"

airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json

@@ -49,5 +49,8 @@
   },
   "tickets": {
     "updatedAt": "2221-10-12T13:37:56.412000+00:00"
+  },
+  "property_history": {
+    "timestamp": 7966012325000
   }
 }