Merge branch 'ddavydov/#20703-source-salesforce-include-pk-in-propert…

…ies-chunks' of github.com:airbytehq/airbyte into ddavydov/#20703-source-salesforce-include-pk-in-properties-chunks

airbyte-cdk/python/.bumpversion.cfg

@@ -3,3 +3,4 @@ current_version = 0.29.0
 commit = False
 
 [bumpversion:file:setup.py]
+[bumpversion:file:Dockerfile]

airbyte-cdk/python/Dockerfile

@@ -0,0 +1,36 @@
+FROM python:3.9.11-alpine3.15 as base
+
+# build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
+
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
+
+# install airbyte-cdk
+RUN pip install --prefix=/install airbyte-cdk==0.29.0
+
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY source_declarative_manifest/main.py ./
+
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
+
+# needs to be the same as CDK
+LABEL io.airbyte.version=0.29.0
+LABEL io.airbyte.name=airbyte/source-declarative-manifest

airbyte-cdk/python/source_declarative_manifest/README.md

@@ -0,0 +1,49 @@
+# Declarative manifest source
+
+This is a generic source that takes the declarative manifest via a key `__injected_declarative_manifest` of its config.
+
+## Execution
+This entrypoint is used for connectors created by the connector builder. These connector's spec is defined in their manifest, which is defined in the config's "__injected_declarative_manifest" field. This allows this entrypoint to be used with any connector manifest.
+
+The spec operation is not supported because the config is not known when running a spec.
+
+## Local development
+
+#### Building
+
+You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow.
+
+To build using Gradle, from the Airbyte repository root, run:
+
+```
+./gradlew airbyte-cdk:python:build
+```
+
+### Locally running the connector
+
+```
+python main.py check --config secrets/config.json
+python main.py discover --config secrets/config.json
+python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json
+```
+
+### Locally running the connector docker image
+
+#### Build
+
+First, make sure you build the latest Docker image:
+```
+./gradlew airbyte-cdk:python:airbyteDocker
+```
+
+The docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in the Dockerfile.
+
+#### Run
+
+Then run any of the connector commands as follows:
+
+```
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-declarative-manifest:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-declarative-manifest:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-declarative-manifest:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
+```

airbyte-cdk/python/source_declarative_manifest/main.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import sys
+from typing import List
+
+from airbyte_cdk.connector import BaseConnector
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
+from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+
+
+def create_manifest(args: List[str]):
+    parsed_args = AirbyteEntrypoint.parse_args(args)
+    if parsed_args.command == "spec":
+        raise ValueError("spec command is not supported for injected declarative manifest")
+
+    config = BaseConnector.read_config(parsed_args.config)
+    if "__injected_declarative_manifest" not in config:
+        raise ValueError(
+            f"Invalid config: `__injected_declarative_manifest` should be provided at the root of the config but config only has keys {list(config.keys())}"
+        )
+    return ManifestDeclarativeSource(config.get("__injected_declarative_manifest"))
+
+
+if __name__ == "__main__":
+    source = create_manifest(sys.argv[1:])
+    launch(source, sys.argv[1:])

airbyte-cdk/python/unit_tests/test_source_declarative_manifest.py

@@ -0,0 +1,98 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import copy
+import json
+
+import pytest
+from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from source_declarative_manifest.main import create_manifest
+
+CONFIG = {
+    "__injected_declarative_manifest": {
+        "version": "0.1.0",
+        "definitions": {
+            "selector": {
+                "extractor": {
+                    "field_path": []
+                }
+            },
+            "requester": {
+                "url_base": "https://test.com/api",
+                "http_method": "GET"
+            },
+            "retriever": {
+                "record_selector": {
+                    "$ref": "#/definitions/selector"
+                },
+                "requester": {
+                    "$ref": "#/definitions/requester"
+                }
+            },
+            "base_stream": {
+                "retriever": {
+                    "$ref": "#/definitions/retriever"
+                }
+            },
+            "data_stream": {
+                "$ref": "#/definitions/base_stream",
+                "$parameters": {
+                    "name": "data",
+                    "path": "/data"
+                }
+            },
+        },
+        "streams": [
+            "#/definitions/data_stream",
+        ],
+        "check": {
+            "stream_names": [
+                "data",
+            ]
+        },
+        "spec": {
+            "type": "Spec",
+            "documentation_url": "https://test.com/doc",
+            "connection_specification": {
+                "$schema": "http://json-schema.org/draft-07/schema#",
+                "title": "Test Spec",
+                "type": "object",
+                "additionalProperties": True,
+                "properties": {}
+            }
+        }
+    }
+}
+
+
+@pytest.fixture
+def valid_config_file(tmp_path):
+    config_file = tmp_path / "config.json"
+    config_file.write_text(json.dumps(CONFIG))
+    return config_file
+
+
+@pytest.fixture
+def config_file_without_injection(tmp_path):
+    config = copy.deepcopy(CONFIG)
+    del config["__injected_declarative_manifest"]
+
+    config_file = tmp_path / "config.json"
+    config_file.write_text(json.dumps(config))
+    return config_file
+
+
+def test_on_spec_command_then_raise_value_error():
+    with pytest.raises(ValueError):
+        create_manifest(["spec"])
+
+
+def test_given_no_injected_declarative_manifest_then_raise_value_error(config_file_without_injection):
+    with pytest.raises(ValueError):
+        create_manifest(["check", "--config", str(config_file_without_injection)])
+
+
+def test_given_injected_declarative_manifest_then_return_declarative_manifest(valid_config_file):
+    source = create_manifest(["check", "--config", str(valid_config_file)])
+    assert isinstance(source, ManifestDeclarativeSource)

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1402,6 +1402,10 @@
   icon: posthog.svg
   sourceType: api
   releaseStage: beta
+  allowedHosts:
+    hosts:
+      - "${base_url}"
+      - "app.posthog.com"
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
@@ -1918,11 +1922,14 @@
 - name: Typeform
   sourceDefinitionId: e7eff203-90bf-43e5-a240-19ea3056c474
   dockerRepository: airbyte/source-typeform
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.com/integrations/sources/typeform
   icon: typeform.svg
   sourceType: api
   releaseStage: beta
+  allowedHosts:
+    hosts:
+      - "api.typeform.com"
 - name: US Census
   sourceDefinitionId: c4cfaeda-c757-489a-8aba-859fb08b6970
   dockerRepository: airbyte/source-us-census

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -15596,7 +15596,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-typeform:0.1.10"
+- dockerImage: "airbyte/source-typeform:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/typeform"
     connectionSpecification:

airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/manifest.yaml.hbs

@@ -5,7 +5,7 @@ definitions:
     type: RecordSelector
     extractor:
       type: DpathExtractor
-      field_pointer: []
+      field_path: []
   requester:
     type: HttpRequester
     url_base: "https://example.com"

airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml

@@ -16,9 +16,13 @@ tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json"
       future_state_path: "integration_tests/future_state.json"
+      cursor_paths:
+        "events": ["2331", "timestamp"]
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json"
       future_state_path: "integration_tests/future_state_old.json"
+      cursor_paths:
+        "events": [ "2331", "timestamp" ]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-typeform/Dockerfile

@@ -12,5 +12,5 @@ COPY main.py ./
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.10
+LABEL io.airbyte.version=0.1.11
 LABEL io.airbyte.name=airbyte/source-typeform

airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml

@@ -23,10 +23,12 @@ acceptance_tests:
           path: "integration_tests/expected_records.jsonl"
   incremental:
     tests:
-      - config_path: "secrets/config.json"
+      - config_path: "secrets/incremental_config.json"
         configured_catalog_path: "integration_tests/configured_catalog_incremental.json"
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"
+        cursor_paths:
+          "responses": ["SdMKQYkv", "submitted_at"]
   full_refresh:
     tests:
       - config_path: "secrets/config.json"