migrate source GA connectors to per-stream states (2) (#17410)

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -330,7 +330,7 @@
 - name: Freshdesk
   sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567
   dockerRepository: airbyte/source-freshdesk
-  dockerImageTag: 0.3.5
+  dockerImageTag: 0.3.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk
   icon: freshdesk.svg
   sourceType: api
@@ -418,7 +418,7 @@
 - name: Google Sheets
   sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7
   dockerRepository: airbyte/source-google-sheets
-  dockerImageTag: 0.2.18
+  dockerImageTag: 0.2.19
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets
   icon: google-sheets.svg
   sourceType: file
@@ -1083,7 +1083,7 @@
 - name: Twilio
   sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1
   dockerRepository: airbyte/source-twilio
-  dockerImageTag: 0.1.9
+  dockerImageTag: 0.1.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/twilio
   icon: twilio.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -3397,7 +3397,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-freshdesk:0.3.5"
+- dockerImage: "airbyte/source-freshdesk:0.3.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk"
     connectionSpecification:
@@ -4354,7 +4354,7 @@
         oauthFlowOutputParameters:
         - - "access_token"
         - - "refresh_token"
-- dockerImage: "airbyte/source-google-sheets:0.2.18"
+- dockerImage: "airbyte/source-google-sheets:0.2.19"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/google-sheets"
     connectionSpecification:
@@ -11403,7 +11403,7 @@
         oauthFlowOutputParameters:
         - - "token"
         - - "key"
-- dockerImage: "airbyte/source-twilio:0.1.9"
+- dockerImage: "airbyte/source-twilio:0.1.10"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/twilio"
     connectionSpecification:

airbyte-integrations/connectors/source-freshdesk/Dockerfile

@@ -34,5 +34,5 @@ COPY source_freshdesk ./source_freshdesk
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.5
+LABEL io.airbyte.version=0.3.6
 LABEL io.airbyte.name=airbyte/source-freshdesk

airbyte-integrations/connectors/source-freshdesk/integration_tests/abnormal_state.json

@@ -1,35 +1,123 @@
-{
-  "agents": {
-    "updated_at": "2121-11-01T00:00:00Z"
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "agents"
+      }
+    }
   },
-  "companies": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "companies"
+      }
+    }
   },
-  "contacts": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "contacts"
+      }
+    }
   },
-  "conversations": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "conversations"
+      }
+    }
   },
-  "groups": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "groups"
+      }
+    }
   },
-  "roles": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "roles"
+      }
+    }
   },
-  "satisfaction_ratings": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "satisfaction_ratings"
+      }
+    }
   },
-  "skills": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "skills"
+      }
+    }
   },
-  "surveys": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "surveys"
+      }
+    }
   },
-  "tickets": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "tickets"
+      }
+    }
   },
-  "time_entries": {
-    "updated_at": "2121-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2121-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "time_entries"
+      }
+    }
   }
-}
+]

airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_state.json

@@ -1,35 +1,123 @@
-{
-  "agents": {
-    "updated_at": "2021-01-01T00:00:00Z"
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-01-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "agents"
+      }
+    }
   },
-  "companies": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "companies"
+      }
+    }
   },
-  "contacts": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "contacts"
+      }
+    }
   },
-  "conversations": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "conversations"
+      }
+    }
   },
-  "groups": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "groups"
+      }
+    }
   },
-  "roles": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "roles"
+      }
+    }
   },
-  "satisfaction_ratings": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "satisfaction_ratings"
+      }
+    }
   },
-  "skills": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "skills"
+      }
+    }
   },
-  "surveys": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "surveys"
+      }
+    }
   },
-  "tickets": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "tickets"
+      }
+    }
   },
-  "time_entries": {
-    "updated_at": "2021-11-01T00:00:00Z"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "updated_at": "2021-11-01T00:00:00Z"
+      },
+      "stream_descriptor": {
+        "name": "time_entries"
+      }
+    }
   }
-}
+]

airbyte-integrations/connectors/source-google-sheets/Dockerfile

@@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.18
+LABEL io.airbyte.version=0.2.19
 LABEL io.airbyte.name=airbyte/source-google-sheets

airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py

@@ -5,13 +5,14 @@
 
 import json
 import socket
-from typing import Dict, Generator
+from typing import Any, Generator, List, MutableMapping, Union
 
 from airbyte_cdk.logger import AirbyteLogger
 from airbyte_cdk.models.airbyte_protocol import (
     AirbyteCatalog,
     AirbyteConnectionStatus,
     AirbyteMessage,
+    AirbyteStateMessage,
     ConfiguredAirbyteCatalog,
     Status,
     Type,
@@ -125,7 +126,11 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog:
             raise Exception(f"Could not run discovery: {reason}")
 
     def read(
-        self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any]
+        self,
+        logger: AirbyteLogger,
+        config: json,
+        catalog: ConfiguredAirbyteCatalog,
+        state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None,
     ) -> Generator[AirbyteMessage, None, None]:
         client = GoogleSheetsClient(self.get_credentials(config))