diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57fbc2cdd4634c..76c1165c56b63e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.50.44 +current_version = 0.50.45 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.github/actions/airbyte-ci-requirements/action.yml b/.github/actions/airbyte-ci-requirements/action.yml index d12a7c1b6b10ac..cb3ae4688c48d3 100644 --- a/.github/actions/airbyte-ci-requirements/action.yml +++ b/.github/actions/airbyte-ci-requirements/action.yml @@ -84,7 +84,7 @@ runs: id: get-dagger-version shell: bash run: | - dagger_version=$(airbyte-ci ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version') + dagger_version=$(airbyte-ci --disable-update-check ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version') echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT" - name: Get runner name diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 8d28ea9e788cd0..55c37cd2e44605 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -83,6 +83,9 @@ inputs: description: "URL to airbyte-ci binary" required: false default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + python_registry_token: + description: "Python registry API token to publish python package" + required: false runs: using: "composite" @@ -154,7 +157,7 @@ runs: shell: bash run: | export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} + airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} env: CI_CONTEXT: "${{ inputs.context }}" CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }} @@ -182,3 +185,4 @@ runs: CI: "True" TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }} DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }} + PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }} diff --git a/.github/workflows/publish-airbyte-lib-command-manually.yml b/.github/workflows/publish-airbyte-lib-command-manually.yml new file mode 100644 index 00000000000000..a4fbfbc8149541 --- /dev/null +++ b/.github/workflows/publish-airbyte-lib-command-manually.yml @@ -0,0 +1,57 @@ +name: Publish AirbyteLib Manually +on: workflow_dispatch + +concurrency: + group: publish-airbyte-lib + cancel-in-progress: false + +jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "publish" + runner_size: "large" + # Getting ci requirements for connectors publish command as there is no special one for poetry publish + airbyte_ci_command: "connectors publish" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} + publish_connectors: + name: Publish airbyte-lib + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Publish + id: publish-airbyte-lib + uses: ./.github/actions/run-dagger-pipeline + with: + context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} + spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: 'poetry --package-path=airbyte-lib publish --registry-url="https://test.pypi.org/legacy/"' + python_registry_token: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index f0d10033f4a2d8..ae431454eda8b0 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -63,6 +63,7 @@ jobs: s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" + python_registry_token: ${{ secrets.PYPI_TOKEN }} - name: Publish connectors [manual] id: publish-connectors @@ -84,6 +85,7 @@ jobs: s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" + python_registry_token: ${{ secrets.PYPI_TOKEN }} set-instatus-incident-on-failure: name: Create Instatus Incident on Failure diff --git a/.gitignore b/.gitignore index b97efca12e817a..bc6841f7b0819a 100644 --- a/.gitignore +++ b/.gitignore @@ -69,13 +69,6 @@ resources/examples/airflow/logs/* # Summary.md keeps getting added and we just don't like it docs/SUMMARY.md -# Files generated by unit tests -**/specs_secrets_mask.yaml - -# Files generated when downloading connector registry -**/init-oss/src/main/resources/seed/oss_registry.json -**/init-oss/src/main/resources/seed/oss_catalog.json - # Output Files generated by scripts lowcode_connector_names.txt num_lowcode_connectors.csv diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 3939f9081a4c61..f08b67beabdcf0 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -166,6 +166,13 @@ MavenLocal debugging steps: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.15.2 | 2024-01-25 | [\#34441](https://github.com/airbytehq/airbyte/pull/34441) | Improve airbyte-api build performance. | +| 0.15.1 | 2024-01-25 | [\#34451](https://github.com/airbytehq/airbyte/pull/34451) | Async destinations: Better logging when we fail to parse an AirbyteMessage | +| 0.15.0 | 2024-01-23 | [\#34441](https://github.com/airbytehq/airbyte/pull/34441) | Removed connector registry and micronaut dependencies. | +| 0.14.2 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Handle case-sensitivity in sentry error grouping | +| 0.14.1 | 2024-01-24 | [\#34468](https://github.com/airbytehq/airbyte/pull/34468) | Add wait for process to be done before ending sync in destination BaseTDTest | +| 0.14.0 | 2024-01-23 | [\#34461](https://github.com/airbytehq/airbyte/pull/34461) | Revert non backward compatible signature changes from 0.13.1 | +| 0.13.3 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Denote if destinations fully support Destinations V2 | | 0.13.2 | 2024-01-18 | [\#34364](https://github.com/airbytehq/airbyte/pull/34364) | Better logging in mongo db source connector | | 0.13.1 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Add postCreateTable hook in destination JdbcSqlGenerator | | 0.13.0 | 2024-01-16 | [\#34177](https://github.com/airbytehq/airbyte/pull/34177) | Add `useExpensiveSafeCasting` param in JdbcSqlGenerator methods; add JdbcTypingDedupingTest fixture; other DV2-related changes | diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle index 0a12cf3048875c..b2def01ec3eee9 100644 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle @@ -9,12 +9,6 @@ java { } dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation group: 'joda-time', name: 'joda-time', version: '2.12.5' implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'com.auth0:java-jwt:3.19.2' @@ -33,11 +27,8 @@ dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor testAnnotationProcessor libs.jmh.annotations - testImplementation libs.bundles.micronaut.test testImplementation 'com.jayway.jsonpath:json-path:2.7.0' testImplementation 'org.mockito:mockito-inline:4.7.0' testImplementation libs.postgresql diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle index 2db31d830e9c21..ae117dba675003 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle @@ -11,58 +11,24 @@ java { } } -def specFile = "$projectDir/src/main/openapi/config.yaml" +String specFile = "$projectDir/src/main/openapi/config.yaml" +String serverOutputDir = "$buildDir/generated/api/server" +String clientOutputDir = "$buildDir/generated/api/client" +String docsOutputDir = "$buildDir/generated/api/docs" +Map schemaMappingsValue = [ + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', + 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', + 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', +] def generate = tasks.register('generate') -// Deprecated -- can be removed once airbyte-server is converted to use the per-domain endpoints generated by generateApiServer -def generateApiServerLegacy = tasks.register('generateApiServerLegacy', GenerateTask) { - def serverOutputDir = "$buildDir/generated/api/server" - - inputs.file specFile - outputs.dir serverOutputDir - - generatorName = "jaxrs-spec" - inputSpec = specFile - outputDir = serverOutputDir - - apiPackage = "io.airbyte.api.generated" - invokerPackage = "io.airbyte.api.invoker.generated" - modelPackage = "io.airbyte.api.model.generated" - - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) - - generateApiDocumentation = false - - configOptions.set([ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true", - /* - JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. - It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. - The below Jackson annotation is made to only keep non null values in serialized json. - We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. - Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. - */ - additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", - ]) -} -generate.configure { - dependsOn generateApiServerLegacy -} - def generateApiServer = tasks.register('generateApiServer', GenerateTask) { - def serverOutputDir = "$buildDir/generated/api/server" inputs.file specFile outputs.dir serverOutputDir @@ -75,16 +41,7 @@ def generateApiServer = tasks.register('generateApiServer', GenerateTask) { invokerPackage = "io.airbyte.api.invoker.generated" modelPackage = "io.airbyte.api.model.generated" - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) + schemaMappings.set(schemaMappingsValue) generateApiDocumentation = false @@ -100,9 +57,6 @@ def generateApiServer = tasks.register('generateApiServer', GenerateTask) { Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. */ additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", - - // Generate separate classes for each endpoint "domain" - useTags: "true" ]) } generate.configure { @@ -110,7 +64,6 @@ generate.configure { } def generateApiClient = tasks.register('generateApiClient', GenerateTask) { - def clientOutputDir = "$buildDir/generated/api/client" inputs.file specFile outputs.dir clientOutputDir @@ -123,16 +76,7 @@ def generateApiClient = tasks.register('generateApiClient', GenerateTask) { invokerPackage = "io.airbyte.api.client.invoker.generated" modelPackage = "io.airbyte.api.client.model.generated" - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) + schemaMappings.set(schemaMappingsValue) library = "native" @@ -149,7 +93,6 @@ generate.configure { } def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) { - def docsOutputDir = "$buildDir/generated/api/docs" generatorName = "html" inputSpec = specFile @@ -159,16 +102,7 @@ def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) { invokerPackage = "io.airbyte.api.client.invoker.generated" modelPackage = "io.airbyte.api.client.model.generated" - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) + schemaMappings.set(schemaMappingsValue) generateApiDocumentation = false @@ -190,9 +124,25 @@ def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) { } } } -generate.configure { +def deleteExistingDocs = tasks.register('deleteOldApiDocs', Delete) { + delete rootProject.file("docs/reference/api/generated-api-html") +} +deleteExistingDocs.configure { dependsOn generateApiDocs } +def copyApiDocs = tasks.register('copyApiDocs', Copy) { + from(docsOutputDir) { + include "**/*.html" + } + into rootProject.file("docs/reference/api/generated-api-html") + includeEmptyDirs = false +} +copyApiDocs.configure { + dependsOn deleteExistingDocs +} +generate.configure { + dependsOn copyApiDocs +} dependencies { implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310' @@ -209,7 +159,11 @@ dependencies { sourceSets { main { java { - srcDirs "$buildDir/generated/api/server/src/gen/java", "$buildDir/generated/api/client/src/main/java", "$projectDir/src/main/java" + srcDirs([ + "$projectDir/src/main/java", + "${serverOutputDir}/src/gen/java", + "${clientOutputDir}/src/main/java", + ]) } resources { srcDir "$projectDir/src/main/openapi/" diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle index ae8a69d7551350..05dc9a3fedb3b1 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle @@ -5,12 +5,6 @@ java { } dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - implementation libs.bundles.micronaut.annotation - testImplementation libs.bundles.micronaut.test - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java index 3d0bcaf9700803..8368cb5128f26d 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java @@ -9,8 +9,6 @@ import io.airbyte.commons.protocol.migrations.MigrationContainer; import io.airbyte.commons.version.Version; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Singleton; import java.util.List; import java.util.Optional; import java.util.Set; @@ -21,7 +19,6 @@ * This class is intended to apply the transformations required to go from one version of the * AirbyteProtocol to another. */ -@Singleton public class AirbyteMessageMigrator { private final MigrationContainer> migrationContainer; @@ -30,7 +27,6 @@ public AirbyteMessageMigrator(final List> migratio migrationContainer = new MigrationContainer<>(migrations); } - @PostConstruct public void initialize() { migrationContainer.initialize(); } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java index 8ead1378bc0226..f045a8cc4a78e6 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java @@ -8,8 +8,6 @@ import io.airbyte.commons.protocol.serde.AirbyteMessageDeserializer; import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; import io.airbyte.commons.version.Version; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Singleton; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -23,7 +21,6 @@ * This class is intended to help access the serializer/deserializer for a given version of the * Airbyte Protocol. */ -@Singleton public class AirbyteMessageSerDeProvider { private final List> deserializersToRegister; @@ -42,7 +39,6 @@ public AirbyteMessageSerDeProvider() { this(Collections.emptyList(), Collections.emptyList()); } - @PostConstruct public void initialize() { deserializersToRegister.forEach(this::registerDeserializer); serializersToRegister.forEach(this::registerSerializer); diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java index 2388e95e4a082a..905fa8294902d7 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java @@ -5,12 +5,10 @@ package io.airbyte.commons.protocol; import io.airbyte.commons.version.Version; -import jakarta.inject.Singleton; /** * Factory to build AirbyteMessageVersionedMigrator */ -@Singleton public class AirbyteProtocolVersionedMigratorFactory { private final AirbyteMessageMigrator airbyteMessageMigrator; diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java index c61522bfad9c18..e62a07f4b772e8 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java @@ -8,12 +8,9 @@ import io.airbyte.commons.protocol.migrations.ConfiguredAirbyteCatalogMigration; import io.airbyte.commons.protocol.migrations.MigrationContainer; import io.airbyte.commons.version.Version; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Singleton; import java.util.List; import java.util.Set; -@Singleton public class ConfiguredAirbyteCatalogMigrator { private final MigrationContainer> migrationContainer; @@ -22,7 +19,6 @@ public ConfiguredAirbyteCatalogMigrator(final List(migrations); } - @PostConstruct public void initialize() { migrationContainer.initialize(); } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java index 4f91e74aa4f010..a64b5201afbf9c 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java @@ -6,9 +6,7 @@ import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; -@Singleton public class AirbyteMessageV0Deserializer extends AirbyteMessageGenericDeserializer { public AirbyteMessageV0Deserializer() { diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java index e50fa17a67fd71..2e28da22301831 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java @@ -6,9 +6,7 @@ import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; -@Singleton public class AirbyteMessageV0Serializer extends AirbyteMessageGenericSerializer { public AirbyteMessageV0Serializer() { diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java index e0708530f17ee4..e85e637b73b3f1 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java @@ -6,9 +6,7 @@ import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; -@Singleton public class AirbyteMessageV1Deserializer extends AirbyteMessageGenericDeserializer { public AirbyteMessageV1Deserializer() { diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java index 2664042d29980b..caf2f0454de1ed 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java @@ -6,9 +6,7 @@ import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; -@Singleton public class AirbyteMessageV1Serializer extends AirbyteMessageGenericSerializer { public AirbyteMessageV1Serializer() { diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java deleted file mode 100644 index 8964e36b6dde0f..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; -import jakarta.inject.Inject; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.junit.jupiter.api.Test; - -@MicronautTest -class AirbyteMessageSerDeProviderMicronautTest { - - @Inject - AirbyteMessageSerDeProvider serDeProvider; - - @Test - void testSerDeInjection() { - // This should contain the list of all the supported majors of the airbyte protocol - final Set expectedVersions = new HashSet<>(List.of("0", "1")); - - assertEquals(expectedVersions, serDeProvider.getDeserializerKeys()); - assertEquals(expectedVersions, serDeProvider.getSerializerKeys()); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java deleted file mode 100644 index 06682838690261..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; -import jakarta.inject.Inject; -import java.util.Set; -import org.junit.jupiter.api.Test; - -@MicronautTest -class MigratorsMicronautTest { - - @Inject - AirbyteMessageMigrator messageMigrator; - - @Inject - ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator; - - // This should contain the list of all the supported majors of the airbyte protocol except the most - // recent one since the migrations themselves are keyed on the lower version. - final Set SUPPORTED_VERSIONS = Set.of(); - - @Test - void testAirbyteMessageMigrationInjection() { - assertEquals(SUPPORTED_VERSIONS, messageMigrator.getMigrationKeys()); - } - - @Test - void testConfiguredAirbyteCatalogMigrationInjection() { - assertEquals(SUPPORTED_VERSIONS, configuredAirbyteCatalogMigrator.getMigrationKeys()); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle index 9073e823a0fe25..c92e3bdac1ecbe 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java-library' - id 'de.undercouch.download' version "5.4.0" } java { @@ -18,10 +17,3 @@ dependencies { // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. implementation 'com.jayway.jsonpath:json-path:2.7.0' } - -def downloadSpecSecretMask = tasks.register('downloadSpecSecretMask', Download) { - src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml' - dest new File(projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml') - overwrite true -} -tasks.named('processResources').configure { dependsOn downloadSpecSecretMask } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 2d0420bf63f03f..0f4c9319750702 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -36,10 +36,14 @@ import java.util.Set; import java.util.function.BiConsumer; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @SuppressWarnings({"PMD.AvoidReassigningParameters", "PMD.AvoidCatchingThrowable"}) public class Jsons { + private static final Logger LOGGER = LoggerFactory.getLogger(Jsons.class); + // Object Mapper is thread-safe private static final ObjectMapper OBJECT_MAPPER = MoreMappers.initMapper(); // sort of a hotfix; I don't know how bad the performance hit is so not turning this on by default @@ -117,7 +121,7 @@ public static Optional tryDeserialize(final String jsonString, final Clas try { return Optional.of(OBJECT_MAPPER.readValue(jsonString, klass)); } catch (final Throwable e) { - return Optional.empty(); + return handleDeserThrowable(e); } } @@ -125,7 +129,7 @@ public static Optional tryDeserializeExact(final String jsonString, final try { return Optional.of(OBJECT_MAPPER_EXACT.readValue(jsonString, klass)); } catch (final Throwable e) { - return Optional.empty(); + return handleDeserThrowable(e); } } @@ -133,7 +137,7 @@ public static Optional tryDeserialize(final String jsonString) { try { return Optional.of(OBJECT_MAPPER.readTree(jsonString)); } catch (final Throwable e) { - return Optional.empty(); + return handleDeserThrowable(e); } } @@ -377,4 +381,33 @@ public DefaultPrettyPrinter withSeparators(final Separators separators) { } + /** + * Simple utility method to log a semi-useful message when deserialization fails. Intentionally + * don't log the actual exception object, because it probably contains some/all of the inputString + * (e.g. `[Source: (String)"{"foo": "bar"; line: 1, column: 13]`). Logging the class name + * can at least help narrow down the problem, without leaking potentially-sensitive information. + */ + private static Optional handleDeserThrowable(Throwable t) { + // Manually build the stacktrace, excluding the top-level exception object + // so that we don't accidentally include the exception message. + // Otherwise we could just do ExceptionUtils.getStackTrace(t). + final StringBuilder sb = new StringBuilder(); + sb.append(t.getClass()); + for (final StackTraceElement traceElement : t.getStackTrace()) { + sb.append("\n\tat "); + sb.append(traceElement.toString()); + } + while (t.getCause() != null) { + t = t.getCause(); + sb.append("\nCaused by "); + sb.append(t.getClass()); + for (final StackTraceElement traceElement : t.getStackTrace()) { + sb.append("\n\tat "); + sb.append(traceElement.toString()); + } + } + LOGGER.warn("Failed to deserialize json due to {}", sb); + return Optional.empty(); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java deleted file mode 100644 index 072884392adc1f..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.logging; - -import com.fasterxml.jackson.core.type.TypeReference; -import io.airbyte.commons.constants.AirbyteSecretConstants; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import java.nio.charset.Charset; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LogEvent; -import org.apache.logging.log4j.core.appender.rewrite.RewritePolicy; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.impl.Log4jLogEvent; -import org.apache.logging.log4j.message.SimpleMessage; -import org.apache.logging.log4j.status.StatusLogger; - -/** - * Custom Log4j2 {@link RewritePolicy} used to intercept all log messages and mask any JSON - * properties in the message that match the list of maskable properties. - *

- * The maskable properties file is generated by a Gradle task in the {@code :airbyte-config:specs} - * project. The file is named {@code specs_secrets_mask.yaml} and is located in the - * {@code src/main/resources/seed} directory of the {@link :airbyte-config:init} project. - */ -@Plugin(name = "MaskedDataInterceptor", - category = "Core", - elementType = "rewritePolicy", - printObject = true) -public class MaskedDataInterceptor implements RewritePolicy { - - protected static final Logger logger = StatusLogger.getLogger(); - - /** - * The pattern used to determine if a message contains sensitive data. - */ - private final Optional pattern; - - @PluginFactory - public static MaskedDataInterceptor createPolicy( - @PluginAttribute(value = "specMaskFile", - defaultString = "/seed/specs_secrets_mask.yaml") final String specMaskFile) { - return new MaskedDataInterceptor(specMaskFile); - } - - private MaskedDataInterceptor(final String specMaskFile) { - this.pattern = buildPattern(specMaskFile); - } - - @Override - public LogEvent rewrite(final LogEvent source) { - return Log4jLogEvent.newBuilder() - .setLoggerName(source.getLoggerName()) - .setMarker(source.getMarker()) - .setLoggerFqcn(source.getLoggerFqcn()) - .setLevel(source.getLevel()) - .setMessage(new SimpleMessage(applyMask(source.getMessage().getFormattedMessage()))) - .setThrown(source.getThrown()) - .setContextMap(source.getContextMap()) - .setContextStack(source.getContextStack()) - .setThreadName(source.getThreadName()) - .setSource(source.getSource()) - .setTimeMillis(source.getTimeMillis()) - .build(); - } - - /** - * Applies the mask to the message, if necessary. - * - * @param message The log message. - * @return The possibly masked log message. - */ - private String applyMask(final String message) { - if (pattern.isPresent()) { - return message.replaceAll(pattern.get(), "\"$1\":\"" + AirbyteSecretConstants.SECRETS_MASK + "\""); - } else { - return message; - } - } - - /** - * Loads the maskable properties from the provided file. - * - * @param specMaskFile The spec mask file. - * @return The set of maskable properties. - */ - private Set getMaskableProperties(final String specMaskFile) { - - try { - final String maskFileContents = IOUtils.toString(getClass().getResourceAsStream(specMaskFile), Charset.defaultCharset()); - final Map> properties = Jsons.object(Yamls.deserialize(maskFileContents), new TypeReference<>() {}); - return properties.getOrDefault("properties", Set.of()); - } catch (final Exception e) { - logger.error("Unable to load mask data from '{}': {}.", specMaskFile, e.getMessage()); - return Set.of(); - } - } - - /** - * Builds the maskable property matching pattern. - * - * @param specMaskFile The spec mask file. - * @return The regular expression pattern used to find maskable properties. - */ - private Optional buildPattern(final String specMaskFile) { - final Set maskableProperties = getMaskableProperties(specMaskFile); - return !maskableProperties.isEmpty() ? Optional.of(generatePattern(maskableProperties)) : Optional.empty(); - } - - /** - * Generates the property matching pattern string from the provided set of properties. - * - * @param properties The set of properties to match. - * @return The generated regular expression pattern used to match the maskable properties. - */ - private String generatePattern(final Set properties) { - final StringBuilder builder = new StringBuilder(); - builder.append("(?i)"); // case insensitive - builder.append("\"("); - builder.append(properties.stream().collect(Collectors.joining("|"))); - builder.append(")\"\\s*:\\s*(\"(?:[^\"\\\\]|\\\\.)*\"|\\[[^]\\[]*]|\\d+)"); - return builder.toString(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml index 22d52667d0ee0c..8eac07440f576e 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml @@ -1,23 +1,12 @@ - - - %d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n + + %d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n - %d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n - + %d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n ${sys:LOG_LEVEL:-${env:LOG_LEVEL:-INFO}} - - ${sys:S3_LOG_BUCKET:-${env:S3_LOG_BUCKET}} - ${sys:S3_LOG_BUCKET_REGION:-${env:S3_LOG_BUCKET_REGION}} - ${sys:AWS_ACCESS_KEY_ID:-${env:AWS_ACCESS_KEY_ID}} - ${sys:AWS_SECRET_ACCESS_KEY:-${env:AWS_SECRET_ACCESS_KEY}} - ${sys:S3_MINIO_ENDPOINT:-${env:S3_MINIO_ENDPOINT}} - ${sys:S3_PATH_STYLE_ACCESS:-${env:S3_PATH_STYLE_ACCESS}} - - ${sys:GCS_LOG_BUCKET:-${env:GCS_LOG_BUCKET}} @@ -34,157 +23,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java deleted file mode 100644 index fc4e8120072eda..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.logging; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.airbyte.commons.io.IOs; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -class Log4j2ConfigTest { - - private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests"); - private static final String LOG_FILENAME = "logs.log"; - private Path root; - - @BeforeEach - void setUp() throws IOException { - root = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "test"); - MDC.clear(); - } - - @Test - void testWorkerDispatch() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - MDC.put("context", "worker"); - MDC.put("job_log_path", root + "/" + LOG_FILENAME); - logger.error("random message testWorkerDispatch"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertTrue(IOs.readFile(root, LOG_FILENAME).contains("random message testWorkerDispatch")); - } - - @Test - void testLogSeparateFiles() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testLogSeparateFiles"); - - final Path root1 = root.resolve("1"); - final Path root2 = root.resolve("2"); - - final ExecutorService executor = Executors.newFixedThreadPool(2); - executor.submit(() -> { - MDC.put("job_log_path", root1 + "/" + LOG_FILENAME); - logger.error("random message 1"); - }); - - executor.submit(() -> { - MDC.put("job_log_path", root2 + "/" + LOG_FILENAME); - logger.error("random message 2"); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertTrue(IOs.readFile(root1, LOG_FILENAME).contains("random message 1")); - assertTrue(IOs.readFile(root2, LOG_FILENAME).contains("random message 2")); - } - - @Test - void testLogNoJobRoot() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - logger.error("random message testLogNoJobRoot"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertFalse(Files.exists(root.resolve(LOG_FILENAME))); - } - - @Test - void testAppDispatch() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testAppDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - MDC.put("workspace_app_root", root.toString()); - logger.error("random message testAppDispatch"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertTrue(IOs.readFile(root, LOG_FILENAME).contains("random message testAppDispatch")); - } - - @Test - void testLogNoAppRoot() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testAppDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - logger.error("random message testLogNoAppRoot"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertFalse(Files.exists(root.resolve(LOG_FILENAME))); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle index b64c0a8c7b4033..f9146068689d8b 100644 --- a/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle @@ -12,9 +12,6 @@ java { } dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') } diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index cfef9562cbbe17..8b35718ef55d18 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -26,7 +26,6 @@ dependencies { compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') testCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java index 64502fb55232f0..56eecef479ef76 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java @@ -71,22 +71,26 @@ public void uncaughtException(final Thread thread, final Throwable throwable) { final Optional deinterpolatableException = ExceptionUtils.getThrowableList(throwable).stream() .filter(t -> THROWABLES_TO_DEINTERPOLATE.stream().anyMatch(deinterpolatableClass -> deinterpolatableClass.isAssignableFrom(t.getClass()))) .findFirst(); + final boolean messageWasMangled; if (deinterpolatableException.isPresent()) { + final String originalMessage = deinterpolatableException.get().getMessage(); mangledMessage = STRINGS_TO_DEINTERPOLATE.stream() // Sort the strings longest to shortest, in case any target string is a substring of another // e.g. "airbyte_internal" should be swapped out before "airbyte" .sorted(Comparator.comparing(String::length).reversed()) - .reduce(deinterpolatableException.get().getMessage(), AirbyteExceptionHandler::deinterpolate); + .reduce(originalMessage, AirbyteExceptionHandler::deinterpolate); + messageWasMangled = !mangledMessage.equals(originalMessage); } else { mangledMessage = throwable.getMessage(); + messageWasMangled = false; } - // If we did not modify the message (either not a deinterpolatable class, or we tried to - // deinterpolate - // but made no changes) then emit our default trace message - if (mangledMessage.equals(throwable.getMessage())) { + if (!messageWasMangled) { + // If we did not modify the message (either not a deinterpolatable class, or we tried to + // deinterpolate but made no changes) then emit our default trace message AirbyteTraceMessageUtility.emitSystemErrorTrace(throwable, logMessage); } else { + // If we did modify the message, then emit a custom trace message AirbyteTraceMessageUtility.emitCustomErrorTrace(throwable.getMessage(), mangledMessage); } @@ -95,7 +99,8 @@ public void uncaughtException(final Thread thread, final Throwable throwable) { @NotNull private static String deinterpolate(final String message, final String targetString) { - final String quotedTarget = '(' + Pattern.quote(targetString) + ')'; + // (?i) makes the pattern case-insensitive + final String quotedTarget = '(' + "(?i)" + Pattern.quote(targetString) + ')'; final String targetRegex = REGEX_PREFIX + quotedTarget + REGEX_SUFFIX; final Pattern pattern = Pattern.compile(targetRegex); final Matcher matcher = pattern.matcher(message); @@ -116,7 +121,7 @@ public static void addThrowableForDeinterpolation(final Class { final JsonNode config = parseConfig(parsed.getConfigPath()); if (integration instanceof Destination) { - DestinationConfig.initialize(config); + DestinationConfig.initialize(config, ((Destination) integration).isV2Destination()); } try { validateConfig(integration.spec().getConnectionSpecification(), config, "CHECK"); @@ -183,7 +183,7 @@ private void runInternal(final IntegrationConfig parsed) throws Exception { final JsonNode config = parseConfig(parsed.getConfigPath()); validateConfig(integration.spec().getConnectionSpecification(), config, "WRITE"); // save config to singleton - DestinationConfig.initialize(config); + DestinationConfig.initialize(config, ((Destination) integration).isV2Destination()); final ConfiguredAirbyteCatalog catalog = parseConfig(parsed.getCatalogPath(), ConfiguredAirbyteCatalog.class); try (final SerializedAirbyteMessageConsumer consumer = destination.getSerializedMessageConsumer(config, catalog, outputRecordCollector)) { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java index b59e757b62c4a4..8820b1d7017f9a 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java @@ -9,11 +9,12 @@ public class TypingAndDedupingFlag { public static boolean isDestinationV2() { - return DestinationConfig.getInstance().getBooleanValue("use_1s1t_format"); + return DestinationConfig.getInstance().getIsV2Destination() + || DestinationConfig.getInstance().getBooleanValue("use_1s1t_format"); } - public static Optional getRawNamespaceOverride(String option) { - String rawOverride = DestinationConfig.getInstance().getTextValue(option); + public static Optional getRawNamespaceOverride(final String option) { + final String rawOverride = DestinationConfig.getInstance().getTextValue(option); if (rawOverride == null || rawOverride.isEmpty()) { return Optional.empty(); } else { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java index 878eef089be030..81d508b0dd2c77 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java @@ -4,14 +4,9 @@ package io.airbyte.cdk.integrations.base.adaptive; -import io.airbyte.cdk.integrations.base.Command; import io.airbyte.cdk.integrations.base.Destination; -import io.airbyte.cdk.integrations.base.DestinationConfig; -import io.airbyte.cdk.integrations.base.IntegrationCliParser; -import io.airbyte.cdk.integrations.base.IntegrationConfig; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,15 +83,6 @@ private Destination getDestination() { } public void run(final String[] args) throws Exception { - // getDestination() sometimes depends on the singleton being initialized. - // Parse the CLI args just so we can accomplish that. - IntegrationConfig parsedArgs = new IntegrationCliParser().parse(args); - if (parsedArgs.getCommand() != Command.SPEC) { - DestinationConfig.initialize(IntegrationRunner.parseConfig(parsedArgs.getConfigPath())); - } else { - DestinationConfig.initialize(Jsons.emptyObject()); - } - final Destination destination = getDestination(); LOGGER.info("Starting destination: {}", destination.getClass().getName()); new IntegrationRunner(destination).run(args); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index b18dfa7feb695e..c85657bf9ac96b 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.13.2 +version=0.15.2 diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java index 25812410a01a56..23f871f0bacde3 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java @@ -64,12 +64,13 @@ void testMessageDeinterpolation() throws Exception { // foo and bar are added to the list explicitly // name and description are added implicitly by the exception handler. // all of them should be replaced by '?' - runTestWithMessage("Error happened in arst_foo_bar_zxcv (name: description)"); + // (including FOO, which should be detected case-insensitively) + runTestWithMessage("Error happened in arst_FOO_bar_zxcv (name: description)"); final AirbyteMessage traceMessage = findFirstTraceMessage(); assertAll( () -> assertEquals(AirbyteTraceMessage.Type.ERROR, traceMessage.getTrace().getType()), - () -> assertEquals("Error happened in arst_foo_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()), + () -> assertEquals("Error happened in arst_FOO_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()), () -> assertEquals("Error happened in arst_?_?_zxcv (?: ?)", traceMessage.getTrace().getError().getInternalMessage()), () -> assertEquals(AirbyteErrorTraceMessage.FailureType.SYSTEM_ERROR, traceMessage.getTrace().getError().getFailureType()), () -> Assertions.assertNull(traceMessage.getTrace().getError().getStackTrace(), diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java index 2d06503baf20d6..68044162bb7249 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java @@ -33,18 +33,21 @@ public void testInitialization() { assertThrows(IllegalStateException.class, DestinationConfig::getInstance); // good initialization - DestinationConfig.initialize(NODE); + DestinationConfig.initialize(NODE, true); assertNotNull(DestinationConfig.getInstance()); assertEquals(NODE, DestinationConfig.getInstance().root); + assertEquals(true, DestinationConfig.getInstance().getIsV2Destination()); // initializing again doesn't change the config final JsonNode nodeUnused = Jsons.deserialize("{}"); - DestinationConfig.initialize(nodeUnused); + DestinationConfig.initialize(nodeUnused, false); assertEquals(NODE, DestinationConfig.getInstance().root); + assertEquals(true, DestinationConfig.getInstance().getIsV2Destination()); } @Test public void testValues() { + DestinationConfig.clearInstance(); DestinationConfig.initialize(NODE); assertEquals("bar", DestinationConfig.getInstance().getTextValue("foo")); @@ -60,6 +63,8 @@ public void testValues() { assertEquals(Jsons.deserialize("\"bar\""), DestinationConfig.getInstance().getNodeValue("foo")); assertEquals(Jsons.deserialize("true"), DestinationConfig.getInstance().getNodeValue("baz")); assertNull(DestinationConfig.getInstance().getNodeValue("blah")); + + assertEquals(false, DestinationConfig.getInstance().getIsV2Destination()); } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index 4668be6e7ccacd..ab74b0eaf9237c 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -19,7 +19,6 @@ dependencies { compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') @@ -27,7 +26,6 @@ dependencies { testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java index b01962a17bc75f..0eef0c5343bf5b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java @@ -45,6 +45,7 @@ public static OnStartFunction onStartFunction(final JdbcDatabase database, final String schema = writeConfig.getOutputSchemaName(); final String stream = writeConfig.getStreamName(); final String dstTableName = writeConfig.getOutputTableName(); + final String stageName = stagingOperations.getStageName(schema, dstTableName); final String stagingPath = stagingOperations.getStagingPath(SerialStagingConsumerFactory.RANDOM_CONNECTION_ID, schema, stream, writeConfig.getOutputTableName(), writeConfig.getWriteDatetime()); @@ -54,7 +55,7 @@ public static OnStartFunction onStartFunction(final JdbcDatabase database, stagingOperations.createSchemaIfNotExists(database, schema); stagingOperations.createTableIfNotExists(database, schema, dstTableName); - stagingOperations.createStageIfNotExists(); + stagingOperations.createStageIfNotExists(database, stageName); /* * When we're in OVERWRITE, clear out the table at the start of a sync, this is an expected side @@ -78,6 +79,7 @@ public static OnStartFunction onStartFunction(final JdbcDatabase database, * upload was unsuccessful */ public static void copyIntoTableFromStage(final JdbcDatabase database, + final String stageName, final String stagingPath, final List stagedFiles, final String tableName, @@ -92,7 +94,7 @@ public static void copyIntoTableFromStage(final JdbcDatabase database, final Lock rawTableInsertLock = typerDeduper.getRawTableInsertLock(streamNamespace, streamName); rawTableInsertLock.lock(); try { - stagingOperations.copyIntoTableFromStage(database, stagingPath, stagedFiles, + stagingOperations.copyIntoTableFromStage(database, stageName, stagingPath, stagedFiles, tableName, schemaName); } finally { rawTableInsertLock.unlock(); @@ -131,6 +133,7 @@ public static OnCloseFunction onCloseFunction(final JdbcDatabase database, for (final WriteConfig writeConfig : writeConfigs) { final String schemaName = writeConfig.getOutputSchemaName(); if (purgeStagingData) { + final String stageName = stagingOperations.getStageName(schemaName, writeConfig.getOutputTableName()); final String stagePath = stagingOperations.getStagingPath( RANDOM_CONNECTION_ID, schemaName, @@ -139,7 +142,9 @@ public static OnCloseFunction onCloseFunction(final JdbcDatabase database, writeConfig.getWriteDatetime()); log.info("Cleaning stage in destination started for stream {}. schema {}, stage: {}", writeConfig.getStreamName(), schemaName, stagePath); - stagingOperations.dropStageIfExists(database, stagePath); + // TODO: This is another weird manifestation of Redshift vs Snowflake using either or variables from + // stageName/StagingPath. + stagingOperations.dropStageIfExists(database, stageName, stagePath); } } typerDeduper.commitFinalTables(); diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialFlush.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialFlush.java index a4cb0c5fdaf38d..767eea2333649e 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialFlush.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialFlush.java @@ -81,14 +81,15 @@ public static FlushBufferFunction function( final WriteConfig writeConfig = pairToWriteConfig.get(pair); final String schemaName = writeConfig.getOutputSchemaName(); + final String stageName = stagingOperations.getStageName(schemaName, writeConfig.getOutputTableName()); final String stagingPath = stagingOperations.getStagingPath( SerialStagingConsumerFactory.RANDOM_CONNECTION_ID, schemaName, writeConfig.getStreamName(), writeConfig.getOutputTableName(), writeConfig.getWriteDatetime()); try (writer) { writer.flush(); - final String stagedFile = stagingOperations.uploadRecordsToStage(database, writer, schemaName, stagingPath); - GeneralStagingFunctions.copyIntoTableFromStage(database, stagingPath, List.of(stagedFile), writeConfig.getOutputTableName(), + final String stagedFile = stagingOperations.uploadRecordsToStage(database, writer, schemaName, stageName, stagingPath); + GeneralStagingFunctions.copyIntoTableFromStage(database, stageName, stagingPath, List.of(stagedFile), writeConfig.getOutputTableName(), schemaName, stagingOperations, writeConfig.getNamespace(), diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java index aac9351b4b7d8a..ae1b132de43faf 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java @@ -15,6 +15,10 @@ * Staging operations focuses on the SQL queries that are needed to success move data into a staging * environment like GCS or S3. In general, the reference of staging is the usage of an object * storage for the purposes of efficiently uploading bulk data to destinations + * + * TODO: This interface is shared between Snowflake and Redshift connectors where the staging + * mechanism is different wire protocol. Make the interface more Generic and have sub interfaces to + * support BlobStorageOperations or Jdbc based staging operations. */ public interface StagingOperations extends SqlOperations { @@ -25,10 +29,19 @@ public interface StagingOperations extends SqlOperations { */ String getStagingPath(UUID connectionId, String namespace, String streamName, String outputTableName, DateTime writeDatetime); + /** + * Returns the staging environment's name + * + * @param namespace Name of schema + * @param streamName Name of the stream + * @return Fully qualified name of the staging environment + */ + String getStageName(String namespace, String streamName); + /** * Create a staging folder where to upload temporary files before loading into the final destination */ - void createStageIfNotExists() throws Exception; + void createStageIfNotExists(JdbcDatabase database, String stageName) throws Exception; /** * Upload the data file into the stage area. @@ -39,7 +52,7 @@ public interface StagingOperations extends SqlOperations { * @param stagingPath path of staging folder to data files * @return the name of the file that was uploaded. */ - String uploadRecordsToStage(JdbcDatabase database, SerializableBuffer recordsData, String schemaName, String stagingPath) + String uploadRecordsToStage(JdbcDatabase database, SerializableBuffer recordsData, String schemaName, String stageName, String stagingPath) throws Exception; /** @@ -52,6 +65,7 @@ String uploadRecordsToStage(JdbcDatabase database, SerializableBuffer recordsDat * @param schemaName name of schema */ void copyIntoTableFromStage(JdbcDatabase database, + String stageName, String stagingPath, List stagedFiles, String tableName, @@ -64,6 +78,6 @@ void copyIntoTableFromStage(JdbcDatabase database, * @param database database used for syncing * @param stageName Name of the staging area used to store files */ - void dropStageIfExists(JdbcDatabase database, String stageName) throws Exception; + void dropStageIfExists(JdbcDatabase database, String stageName, String stagingPath) throws Exception; } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java index 6aa6f1e6e05503..d2aa70a1449c41 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -35,9 +35,7 @@ import io.airbyte.configoss.StandardCheckConnectionInput; import io.airbyte.configoss.StandardCheckConnectionOutput; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import io.airbyte.configoss.StandardDestinationDefinition; import io.airbyte.configoss.WorkerDestinationConfig; -import io.airbyte.configoss.init.LocalDefinitionsProvider; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -65,6 +63,8 @@ import io.airbyte.workers.process.DockerProcessFactory; import io.airbyte.workers.process.ProcessFactory; import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -74,7 +74,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.UUID; @@ -142,20 +141,31 @@ private String getImageNameWithoutTag() { return getImageName().contains(":") ? getImageName().split(":")[0] : getImageName(); } - protected static Optional getOptionalDestinationDefinitionFromProvider( - final String imageNameWithoutTag) { - final LocalDefinitionsProvider provider = new LocalDefinitionsProvider(); - return provider.getDestinationDefinitions().stream() - .filter(definition -> imageNameWithoutTag.equalsIgnoreCase(definition.getDockerRepository())) - .findFirst(); + private JsonNode readMetadata() { + try { + return Jsons.jsonNodeFromFile(MoreResources.readResourceAsFile("metadata.yaml")); + } catch (IllegalArgumentException | URISyntaxException e) { + // Resource is not found. + return Jsons.emptyObject(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } protected String getNormalizationImageName() { - return getOptionalDestinationDefinitionFromProvider(getDestinationDefinitionKey()) - .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig())) - .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationConfig().getNormalizationRepository() + ":" - + NORMALIZATION_VERSION) - .orElse(null); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return null; + } + var normalizationConfig = metadata.get("normalizationConfig"); + if (normalizationConfig == null) { + return null; + } + var normalizationRepository = normalizationConfig.get("normalizationRepository"); + if (normalizationRepository == null) { + return null; + } + return normalizationRepository.asText() + ":" + NORMALIZATION_VERSION; } /** @@ -240,18 +250,24 @@ protected boolean implementsAppend() throws TestHarnessException { } protected boolean normalizationFromDefinition() { - return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag()) - .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig())) - .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig().getNormalizationRepository()) - && Objects.nonNull(standardDestinationDefinition.getNormalizationConfig().getNormalizationTag())) - .orElse(false); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return false; + } + var normalizationConfig = metadata.get("normalizationConfig"); + if (normalizationConfig == null) { + return false; + } + return normalizationConfig.has("normalizationRepository") && normalizationConfig.has("normalizationTag"); } protected boolean dbtFromDefinition() { - return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag()) - .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getSupportsDbt()) - && standardDestinationDefinition.getSupportsDbt()) - .orElse(false); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return false; + } + var supportsDbt = metadata.get("supportsDbt"); + return supportsDbt != null && supportsDbt.asBoolean(false); } protected String getDestinationDefinitionKey() { @@ -259,10 +275,19 @@ protected String getDestinationDefinitionKey() { } protected String getNormalizationIntegrationType() { - return getOptionalDestinationDefinitionFromProvider(getDestinationDefinitionKey()) - .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig())) - .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationConfig().getNormalizationIntegrationType()) - .orElse(null); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return null; + } + var normalizationConfig = metadata.get("normalizationConfig"); + if (normalizationConfig == null) { + return null; + } + var normalizationIntegrationType = normalizationConfig.get("normalizationIntegrationType"); + if (normalizationIntegrationType == null) { + return null; + } + return normalizationIntegrationType.asText(); } /** diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle index a8842db3aa3174..f7b5100887f03b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle @@ -32,12 +32,6 @@ project.configurations { testFixturesRuntimeOnly.extendsFrom runtimeOnly } -configurations.all { - // From airbyte-test-utils - exclude group: 'io.micronaut.jaxrs' - exclude group: 'io.micronaut.sql' -} - // Convert yaml to java: relationaldb.models jsonSchema2Pojo { sourceType = SourceType.YAMLSCHEMA @@ -60,7 +54,6 @@ dependencies { compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') @@ -69,7 +62,6 @@ dependencies { testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg deleted file mode 100644 index 554d6822485077..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle index c76851730e113d..b41a176dc90510 100644 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle @@ -1,28 +1,5 @@ -plugins { - id 'java-library' - id "de.undercouch.download" version "5.4.0" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation 'commons-cli:commons-cli:1.4' - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - implementation libs.lombok - implementation libs.micronaut.cache.caffeine +// TODO: remove this module stub entirely during CDK module consolidation - testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' -} - -def downloadConnectorRegistry = tasks.register('downloadConnectorRegistry', Download) { - src 'https://connectors.airbyte.com/files/registries/v0/oss_registry.json' - dest new File(projectDir, 'src/main/resources/seed/oss_registry.json') - overwrite true -} -tasks.named('processResources')configure { - dependsOn downloadConnectorRegistry +plugins { + id "java-library" } diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/readme.md b/airbyte-cdk/java/airbyte-cdk/init-oss/readme.md deleted file mode 100644 index f9b088d33d6fee..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/readme.md +++ /dev/null @@ -1,12 +0,0 @@ -# airbyte-config:init - -This module fulfills two responsibilities: -1. It is where we declare what connectors should ship with the Platform. See below for more instruction on how it works. -2. It contains the scripts and Dockerfile that allow the `docker-compose` version of Airbyte to mount the local filesystem. This is helpful in cases where a user wants to use a connector that interacts with (reads data from or writes data to) the local filesystem. e.g. `destination-local-json`. - -## The Connector Registry and Spec Secret Masks -The connector registry (list of available connectors) is downloaded at runtime from https://connectors.airbyte.com/files/registries/v0/oss_registry.json - -The spec secret mask (one of the multiple systems that hide your secrets from the logs) is also downloaded at runtime from https://connectors.airbyte.com/files/registries/v0/spec_secret_mask.json - -The logic inside the folder is responsible for downloading these files and making them available to the platform. \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh b/airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh deleted file mode 100755 index c27ee40a1c0c12..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env sh - -set -e - -# hack: attempt to get local mounts to work properly -# constraints: ROOT_PARENT better exist on the local filesystem. -# check that the given directory (ROOT) that we plan to use as a mount -# in other containers is a in fact a directory within the parent. if it -# is, then we make sure it is created. we do this by removing the common -# part of the path from the root and appending it to the mount. then we -# make the directories. -# e.g. ROOT_PARENT=/tmp, ROOT=/tmp/airbyte_local MOUNT=/local_parent. -# We create MOUNT_ROOT which will look like /local_parent/airbyte_local. -# Because it is using the mount name, we can create it on the local -# fileystem from within the container. -MOUNT=$1; echo "MOUNT: $MOUNT" -ROOT_PARENT=$2; echo "ROOT_PARENT: $ROOT_PARENT" -ROOT=$3; echo "ROOT: $ROOT" - -[[ "${ROOT}"="${ROOT_PARENT}"* ]] || (echo "ROOT ${ROOT} is not a child of ROOT_PARENT ${ROOT_PARENT}." && exit 1) -MOUNT_ROOT=${MOUNT}/$(echo $ROOT | sed -e "s|${ROOT_PARENT}||g") -echo "MOUNT_ROOT: ${MOUNT_ROOT}" -mkdir -p ${MOUNT_ROOT} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java deleted file mode 100644 index 66033a188e8c3f..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -public class ConfigNotFoundException extends Exception { - - private static final long serialVersionUID = 836273627; - private final String type; - private final String configId; - - public ConfigNotFoundException(final String type, final String configId) { - super(String.format("config type: %s id: %s", type, configId)); - this.type = type; - this.configId = configId; - } - - public String getType() { - return type; - } - - public String getConfigId() { - return configId; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java deleted file mode 100644 index 34bd03a801c6ee..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.util.List; -import java.util.UUID; - -public interface DefinitionsProvider { - - StandardSourceDefinition getSourceDefinition(final UUID definitionId) throws ConfigNotFoundException; - - List getSourceDefinitions(); - - StandardDestinationDefinition getDestinationDefinition(final UUID definitionId) throws ConfigNotFoundException; - - List getDestinationDefinitions(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java deleted file mode 100644 index 2aca382f70ba09..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.BooleanNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -public class JsonDefinitionsHelper { - - public static JsonNode addMissingTombstoneField(final JsonNode definitionJson) { - final JsonNode currTombstone = definitionJson.get("tombstone"); - if (currTombstone == null || currTombstone.isNull()) { - ((ObjectNode) definitionJson).set("tombstone", BooleanNode.FALSE); - } - return definitionJson; - } - - public static JsonNode addMissingPublicField(final JsonNode definitionJson) { - final JsonNode currPublic = definitionJson.get("public"); - if (currPublic == null || currPublic.isNull()) { - // definitions loaded from seed yamls are by definition public - ((ObjectNode) definitionJson).set("public", BooleanNode.TRUE); - } - return definitionJson; - } - - public static JsonNode addMissingCustomField(final JsonNode definitionJson) { - final JsonNode currCustom = definitionJson.get("custom"); - if (currCustom == null || currCustom.isNull()) { - // definitions loaded from seed yamls are by definition not custom - ((ObjectNode) definitionJson).set("custom", BooleanNode.FALSE); - } - return definitionJson; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java deleted file mode 100644 index 33966a42ad584d..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import com.google.common.io.Resources; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.configoss.CatalogDefinitionsConfig; -import io.airbyte.configoss.CombinedConnectorCatalog; -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -/** - * This provider contains all definitions according to the local catalog json files. - */ -final public class LocalDefinitionsProvider implements DefinitionsProvider { - - private static final String LOCAL_CONNECTOR_REGISTRY_PATH = CatalogDefinitionsConfig.getLocalConnectorCatalogPath(); - - public CombinedConnectorCatalog getLocalDefinitionCatalog() { - try { - final URL url = Resources.getResource(LOCAL_CONNECTOR_REGISTRY_PATH); - final String jsonString = Resources.toString(url, StandardCharsets.UTF_8); - final CombinedConnectorCatalog catalog = Jsons.deserialize(jsonString, CombinedConnectorCatalog.class); - return catalog; - - } catch (final Exception e) { - throw new RuntimeException("Failed to fetch local catalog definitions", e); - } - } - - public Map getSourceDefinitionsMap() { - final CombinedConnectorCatalog catalog = getLocalDefinitionCatalog(); - return catalog.getSources().stream().collect(Collectors.toMap( - StandardSourceDefinition::getSourceDefinitionId, - source -> source.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault(source.getSpec() != null ? source.getSpec().getProtocolVersion() : null).serialize()))); - } - - public Map getDestinationDefinitionsMap() { - final CombinedConnectorCatalog catalog = getLocalDefinitionCatalog(); - return catalog.getDestinations().stream().collect( - Collectors.toMap( - StandardDestinationDefinition::getDestinationDefinitionId, - destination -> destination.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault( - destination.getSpec() != null - ? destination.getSpec().getProtocolVersion() - : null) - .serialize()))); - } - - @Override - public StandardSourceDefinition getSourceDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardSourceDefinition definition = getSourceDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("local_registry:source_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getSourceDefinitions() { - return new ArrayList<>(getSourceDefinitionsMap().values()); - } - - @Override - public StandardDestinationDefinition getDestinationDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardDestinationDefinition definition = getDestinationDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("local_registry:destination_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getDestinationDefinitions() { - return new ArrayList<>(getDestinationDefinitionsMap().values()); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java deleted file mode 100644 index 4d6145a5f35e53..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -/** - * Defines any additional tasks that should be executed after successful boostrapping of the Airbyte - * environment. - */ -public interface PostLoadExecutor { - - /** - * Executes the additional post bootstrapping tasks. - * - * @throws Exception if unable to perform the additional tasks. - */ - void execute() throws Exception; - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java deleted file mode 100644 index c1e4816367a370..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.configoss.CombinedConnectorCatalog; -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import io.micronaut.cache.annotation.CacheConfig; -import io.micronaut.cache.annotation.Cacheable; -import io.micronaut.context.annotation.Primary; -import io.micronaut.context.annotation.Requires; -import io.micronaut.context.annotation.Value; -import jakarta.inject.Singleton; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; - -/** - * This provider pulls the definitions from a remotely hosted catalog. - */ -@Singleton -@Primary -@Requires(property = "airbyte.platform.remote-connector-catalog.url", - notEquals = "") -@CacheConfig("remote-definitions-provider") -@Slf4j -public class RemoteDefinitionsProvider implements DefinitionsProvider { - - private static final HttpClient httpClient = HttpClient.newHttpClient(); - private final URI remoteDefinitionCatalogUrl; - private final Duration timeout; - - public RemoteDefinitionsProvider(@Value("${airbyte.platform.remote-connector-catalog.url}") final String remoteCatalogUrl, - @Value("${airbyte.platform.remote-connector-catalog.timeout-ms}") final long remoteCatalogTimeoutMs) - throws URISyntaxException { - log.info("Creating remote definitions provider for URL '{}'...", remoteCatalogUrl); - remoteDefinitionCatalogUrl = new URI(remoteCatalogUrl); - timeout = Duration.ofMillis(remoteCatalogTimeoutMs); - } - - private Map getSourceDefinitionsMap() { - final CombinedConnectorCatalog catalog = getRemoteDefinitionCatalog(); - return catalog.getSources().stream().collect(Collectors.toMap( - StandardSourceDefinition::getSourceDefinitionId, - source -> source.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault(source.getSpec() != null ? source.getSpec().getProtocolVersion() : null).serialize()))); - } - - private Map getDestinationDefinitionsMap() { - final CombinedConnectorCatalog catalog = getRemoteDefinitionCatalog(); - return catalog.getDestinations().stream().collect(Collectors.toMap( - StandardDestinationDefinition::getDestinationDefinitionId, - destination -> destination.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault(destination.getSpec() != null ? destination.getSpec().getProtocolVersion() : null).serialize()))); - } - - @Override - public StandardSourceDefinition getSourceDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardSourceDefinition definition = getSourceDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("remote_registry:source_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getSourceDefinitions() { - return new ArrayList<>(getSourceDefinitionsMap().values()); - } - - @Override - public StandardDestinationDefinition getDestinationDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardDestinationDefinition definition = getDestinationDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("remote_registry:destination_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getDestinationDefinitions() { - return new ArrayList<>(getDestinationDefinitionsMap().values()); - } - - @Cacheable - public CombinedConnectorCatalog getRemoteDefinitionCatalog() { - try { - final HttpRequest request = HttpRequest.newBuilder(remoteDefinitionCatalogUrl).timeout(timeout).header("accept", "application/json").build(); - - final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - if (errorStatusCode(response)) { - throw new IOException( - "getRemoteDefinitionCatalog request ran into status code error: " + response.statusCode() + " with message: " + response.getClass()); - } - - log.info("Fetched latest remote definitions ({})", response.body().hashCode()); - return Jsons.deserialize(response.body(), CombinedConnectorCatalog.class); - } catch (final Exception e) { - throw new RuntimeException("Failed to fetch remote definitions", e); - } - } - - private static Boolean errorStatusCode(final HttpResponse response) { - return response.statusCode() >= 400; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json deleted file mode 100644 index 75a2b686942294..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "2817b3f0-04e4-4c7a-9f32-7a5e8a83db95", - "name": "PagerDuty", - "dockerRepository": "farosai/airbyte-pagerduty-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/pagerduty", - "icon": "pagerduty.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json deleted file mode 100644 index 80c10bf0b68001..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "6fe89830-d04d-401b-aad6-6552ffa5c4af", - "name": "Harness", - "dockerRepository": "farosai/airbyte-harness-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/harness", - "icon": "harness.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json deleted file mode 100644 index 1ec101ea71ecf6..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "7e20ce3e-d820-4327-ad7a-88f3927fd97a", - "name": "VictorOps", - "dockerRepository": "farosai/airbyte-victorops-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/victorops", - "icon": "victorops.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json deleted file mode 100644 index ada2c538021705..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "c47d6804-8b98-449f-970a-5ddb5cb5d7aa", - "name": "Customer.io", - "dockerRepository": "farosai/airbyte-customer-io-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/customer-io", - "icon": "customer-io.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg deleted file mode 100644 index 36c7f62a33c171..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg deleted file mode 100644 index 01e394f5b30f34..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java deleted file mode 100644 index 768ea436aa5307..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.UUID; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -class LocalDefinitionsProviderTest { - - private static LocalDefinitionsProvider localDefinitionsProvider; - - @BeforeAll - static void setup() throws IOException { - localDefinitionsProvider = new LocalDefinitionsProvider(); - } - - @Test - void testGetSourceDefinition() throws Exception { - // source - final UUID stripeSourceId = UUID.fromString("e094cb9a-26de-4645-8761-65c0c425d1de"); - final StandardSourceDefinition stripeSource = localDefinitionsProvider.getSourceDefinition(stripeSourceId); - assertEquals(stripeSourceId, stripeSource.getSourceDefinitionId()); - assertEquals("Stripe", stripeSource.getName()); - assertEquals("airbyte/source-stripe", stripeSource.getDockerRepository()); - assertEquals("https://docs.airbyte.com/integrations/sources/stripe", stripeSource.getDocumentationUrl()); - assertEquals("stripe.svg", stripeSource.getIcon()); - assertEquals(URI.create("https://docs.airbyte.com/integrations/sources/stripe"), stripeSource.getSpec().getDocumentationUrl()); - assertEquals(false, stripeSource.getTombstone()); - assertEquals("0.2.0", stripeSource.getProtocolVersion()); - } - - @Test - @SuppressWarnings({"PMD.AvoidDuplicateLiterals"}) - void testGetDestinationDefinition() throws Exception { - final UUID s3DestinationId = UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362"); - final StandardDestinationDefinition s3Destination = localDefinitionsProvider - .getDestinationDefinition(s3DestinationId); - assertEquals(s3DestinationId, s3Destination.getDestinationDefinitionId()); - assertEquals("S3", s3Destination.getName()); - assertEquals("airbyte/destination-s3", s3Destination.getDockerRepository()); - assertEquals("https://docs.airbyte.com/integrations/destinations/s3", s3Destination.getDocumentationUrl()); - assertEquals(URI.create("https://docs.airbyte.com/integrations/destinations/s3"), s3Destination.getSpec().getDocumentationUrl()); - assertEquals(false, s3Destination.getTombstone()); - assertEquals("0.2.0", s3Destination.getProtocolVersion()); - } - - @Test - void testGetInvalidDefinitionId() { - final UUID invalidDefinitionId = UUID.fromString("1a7c360c-1289-4b96-a171-2ac1c86fb7ca"); - - assertThrows( - ConfigNotFoundException.class, - () -> localDefinitionsProvider.getSourceDefinition(invalidDefinitionId)); - assertThrows( - ConfigNotFoundException.class, - () -> localDefinitionsProvider.getDestinationDefinition(invalidDefinitionId)); - } - - @Test - void testGetSourceDefinitions() { - final List sourceDefinitions = localDefinitionsProvider.getSourceDefinitions(); - assertFalse(sourceDefinitions.isEmpty()); - assertTrue(sourceDefinitions.stream().allMatch(sourceDef -> sourceDef.getProtocolVersion().length() > 0)); - } - - @Test - void testGetDestinationDefinitions() { - final List destinationDefinitions = localDefinitionsProvider.getDestinationDefinitions(); - assertFalse(destinationDefinitions.isEmpty()); - assertTrue(destinationDefinitions.stream().allMatch(sourceDef -> sourceDef.getProtocolVersion().length() > 0)); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java deleted file mode 100644 index 3cfcc905947b35..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.io.Resources; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.net.http.HttpTimeoutException; -import java.nio.charset.Charset; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class RemoteDefinitionsProviderTest { - - private MockWebServer webServer; - private MockResponse validCatalogResponse; - private String catalogUrl; - private JsonNode jsonCatalog; - - @BeforeEach - void setup() throws IOException { - webServer = new MockWebServer(); - catalogUrl = webServer.url("/connector_catalog.json").toString(); - - final URL testCatalog = Resources.getResource("connector_catalog.json"); - final String jsonBody = Resources.toString(testCatalog, Charset.defaultCharset()); - jsonCatalog = Jsons.deserialize(jsonBody); - validCatalogResponse = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "application/json; charset=utf-8") - .addHeader("Cache-Control", "no-cache") - .setBody(jsonBody); - } - - @Test - @SuppressWarnings({"PMD.AvoidDuplicateLiterals"}) - void testGetSourceDefinition() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final UUID stripeSourceId = UUID.fromString("e094cb9a-26de-4645-8761-65c0c425d1de"); - final StandardSourceDefinition stripeSource = remoteDefinitionsProvider.getSourceDefinition(stripeSourceId); - assertEquals(stripeSourceId, stripeSource.getSourceDefinitionId()); - assertEquals("Stripe", stripeSource.getName()); - assertEquals("airbyte/source-stripe", stripeSource.getDockerRepository()); - assertEquals("https://docs.airbyte.io/integrations/sources/stripe", stripeSource.getDocumentationUrl()); - assertEquals("stripe.svg", stripeSource.getIcon()); - assertEquals(URI.create("https://docs.airbyte.io/integrations/sources/stripe"), stripeSource.getSpec().getDocumentationUrl()); - assertEquals(false, stripeSource.getTombstone()); - assertEquals("0.2.1", stripeSource.getProtocolVersion()); - } - - @Test - @SuppressWarnings({"PMD.AvoidDuplicateLiterals"}) - void testGetDestinationDefinition() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final UUID s3DestinationId = UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362"); - final StandardDestinationDefinition s3Destination = remoteDefinitionsProvider - .getDestinationDefinition(s3DestinationId); - assertEquals(s3DestinationId, s3Destination.getDestinationDefinitionId()); - assertEquals("S3", s3Destination.getName()); - assertEquals("airbyte/destination-s3", s3Destination.getDockerRepository()); - assertEquals("https://docs.airbyte.io/integrations/destinations/s3", s3Destination.getDocumentationUrl()); - assertEquals(URI.create("https://docs.airbyte.io/integrations/destinations/s3"), s3Destination.getSpec().getDocumentationUrl()); - assertEquals(false, s3Destination.getTombstone()); - assertEquals("0.2.2", s3Destination.getProtocolVersion()); - } - - @Test - void testGetInvalidDefinitionId() throws Exception { - webServer.enqueue(validCatalogResponse); - webServer.enqueue(validCatalogResponse); - - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final UUID invalidDefinitionId = UUID.fromString("1a7c360c-1289-4b96-a171-2ac1c86fb7ca"); - - assertThrows( - ConfigNotFoundException.class, - () -> remoteDefinitionsProvider.getSourceDefinition(invalidDefinitionId)); - assertThrows( - ConfigNotFoundException.class, - () -> remoteDefinitionsProvider.getDestinationDefinition(invalidDefinitionId)); - } - - @Test - void testGetSourceDefinitions() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final List sourceDefinitions = remoteDefinitionsProvider.getSourceDefinitions(); - final int expectedNumberOfSources = MoreIterators.toList(jsonCatalog.get("sources").elements()).size(); - assertEquals(expectedNumberOfSources, sourceDefinitions.size()); - assertTrue(sourceDefinitions.stream().allMatch(sourceDef -> sourceDef.getProtocolVersion().length() > 0)); - } - - @Test - void testGetDestinationDefinitions() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final List destinationDefinitions = remoteDefinitionsProvider.getDestinationDefinitions(); - final int expectedNumberOfDestinations = MoreIterators.toList(jsonCatalog.get("destinations").elements()).size(); - assertEquals(expectedNumberOfDestinations, destinationDefinitions.size()); - assertTrue(destinationDefinitions.stream().allMatch(destDef -> destDef.getProtocolVersion().length() > 0)); - } - - @Test - void testBadResponseStatus() { - webServer.enqueue(new MockResponse().setResponseCode(404)); - final RuntimeException ex = assertThrows(RuntimeException.class, () -> { - new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(1)).getDestinationDefinitions(); - }); - - assertTrue(ex.getMessage().contains("Failed to fetch remote definitions")); - assertTrue(ex.getCause() instanceof IOException); - } - - @Test - void testTimeOut() { - // No request enqueued -> Timeout - final RuntimeException ex = assertThrows(RuntimeException.class, () -> { - new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(1)).getDestinationDefinitions(); - }); - - assertTrue(ex.getMessage().contains("Failed to fetch remote definitions")); - assertTrue(ex.getCause() instanceof HttpTimeoutException); - } - - @Test - void testNonJson() { - final MockResponse notJson = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "application/json; charset=utf-8") - .addHeader("Cache-Control", "no-cache") - .setBody("not json"); - webServer.enqueue(notJson); - assertThrows(RuntimeException.class, () -> { - new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(1)).getDestinationDefinitions(); - }); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java deleted file mode 100644 index 68136d8bfef24e..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.JsonSchemas; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -@Slf4j -class SpecFormatTest { - - @Test - void testOnAllExistingConfig() throws IOException, JsonValidationException { - final DefinitionsProvider definitionsProvider = new LocalDefinitionsProvider(); - - final List sourceSpecs = definitionsProvider.getSourceDefinitions() - .stream() - .map(standardSourceDefinition -> standardSourceDefinition.getSpec().getConnectionSpecification()) - .toList(); - - final List destinationSpecs = definitionsProvider.getDestinationDefinitions() - .stream() - .map(standardDestinationDefinition -> standardDestinationDefinition.getSpec().getConnectionSpecification()) - .toList(); - - final List allSpecs = new ArrayList<>(); - - allSpecs.addAll(sourceSpecs); - allSpecs.addAll(destinationSpecs); - - Assertions.assertThat(allSpecs) - .flatMap(spec -> { - try { - if (!isValidJsonSchema(spec)) { - throw new RuntimeException("Fail JsonSecretsProcessor validation"); - } - JsonSchemas.traverseJsonSchema(spec, (node, path) -> {}); - return Collections.emptyList(); - } catch (final Exception e) { - log.error("failed on: " + spec.toString(), e); - return List.of(e); - } - }) - .isEmpty(); - } - - private static boolean isValidJsonSchema(final JsonNode schema) { - return schema.isObject() && ((schema.has("properties") && schema.get("properties").isObject()) - || (schema.has("oneOf") && schema.get("oneOf").isArray())); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json deleted file mode 100644 index f4e09b0d95c525..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json +++ /dev/null @@ -1,14185 +0,0 @@ -{ - "destinations": [ - { - "destinationDefinitionId": "0eeee7fb-518f-4045-bacc-9619e31c43ea", - "name": "Amazon SQS", - "dockerRepository": "airbyte/destination-amazon-sqs", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/amazon-sqs", - "icon": "amazonsqs.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/amazon-sqs", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Amazon Sqs", - "type": "object", - "required": ["queue_url", "region"], - "additionalProperties": false, - "properties": { - "queue_url": { - "title": "Queue URL", - "description": "URL of the SQS Queue", - "type": "string", - "examples": [ - "https://sqs.eu-west-1.amazonaws.com/1234567890/my-example-queue" - ], - "order": 0 - }, - "region": { - "title": "AWS Region", - "description": "AWS Region of the SQS Queue", - "type": "string", - "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 1 - }, - "message_delay": { - "title": "Message Delay", - "description": "Modify the Message Delay of the individual message from the Queue's default (seconds).", - "type": "integer", - "examples": ["15"], - "order": 2 - }, - "access_key": { - "title": "AWS IAM Access Key ID", - "description": "The Access Key ID of the AWS IAM Role to use for sending messages", - "type": "string", - "examples": ["xxxxxHRNxxx3TBxxxxxx"], - "order": 3, - "airbyte_secret": true - }, - "secret_key": { - "title": "AWS IAM Secret Key", - "description": "The Secret Key of the AWS IAM Role to use for sending messages", - "type": "string", - "examples": ["hu+qE5exxxxT6o/ZrKsxxxxxxBhxxXLexxxxxVKz"], - "order": 4, - "airbyte_secret": true - }, - "message_body_key": { - "title": "Message Body Key", - "description": "Use this property to extract the contents of the named key in the input record to use as the SQS message body. If not set, the entire content of the input record data is used as the message body.", - "type": "string", - "examples": ["myDataPath"], - "order": 5 - }, - "message_group_id": { - "title": "Message Group Id", - "description": "The tag that specifies that a message belongs to a specific message group. This parameter applies only to, and is REQUIRED by, FIFO queues.", - "type": "string", - "examples": ["my-fifo-group"], - "order": 6 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "b4c5d105-31fd-4817-96b6-cb923bfc04cb", - "name": "Azure Blob Storage", - "dockerRepository": "airbyte/destination-azure-blob-storage", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/azureblobstorage", - "icon": "azureblobstorage.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/azureblobstorage", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AzureBlobStorage Destination Spec", - "type": "object", - "required": [ - "azure_blob_storage_account_name", - "azure_blob_storage_account_key", - "format" - ], - "additionalProperties": false, - "properties": { - "azure_blob_storage_endpoint_domain_name": { - "title": "Endpoint Domain Name", - "type": "string", - "default": "blob.core.windows.net", - "description": "This is Azure Blob Storage endpoint domain name. Leave default value (or leave it empty if run container from command line) to use Microsoft native from example.", - "examples": ["blob.core.windows.net"] - }, - "azure_blob_storage_container_name": { - "title": "Azure blob storage container (Bucket) Name", - "type": "string", - "description": "The name of the Azure blob storage container. If not exists - will be created automatically. May be empty, then will be created automatically airbytecontainer+timestamp", - "examples": ["airbytetescontainername"] - }, - "azure_blob_storage_account_name": { - "title": "Azure Blob Storage account name", - "type": "string", - "description": "The account's name of the Azure Blob Storage.", - "examples": ["airbyte5storage"] - }, - "azure_blob_storage_account_key": { - "title": "Azure Blob Storage account key", - "description": "The Azure blob storage account key.", - "airbyte_secret": true, - "type": "string", - "examples": [ - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd==" - ] - }, - "azure_blob_storage_output_buffer_size": { - "title": "Azure Blob Storage output buffer size (Megabytes)", - "type": "integer", - "description": "The amount of megabytes to buffer for the output stream to Azure. This will impact memory footprint on workers, but may need adjustment for performance and appropriate block size in Azure.", - "minimum": 1, - "maximum": 2047, - "default": 5, - "examples": [5] - }, - "format": { - "title": "Output Format", - "type": "object", - "description": "Output data format", - "oneOf": [ - { - "title": "CSV: Comma-Separated Values", - "required": ["format_type", "flattening"], - "properties": { - "format_type": { - "type": "string", - "const": "CSV" - }, - "flattening": { - "type": "string", - "title": "Normalization (Flattening)", - "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - } - } - }, - { - "title": "JSON Lines: newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "const": "JSONL" - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", - "name": "BigQuery", - "dockerRepository": "airbyte/destination-bigquery", - "dockerImageTag": "1.1.15", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "icon": "bigquery.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Destination Spec", - "type": "object", - "required": ["project_id", "dataset_location", "dataset_id"], - "additionalProperties": true, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", - "title": "Project ID", - "order": 0 - }, - "dataset_location": { - "type": "string", - "description": "The location of the dataset. Warning: Changes made after creation will not be applied. Read more here.", - "title": "Dataset Location", - "order": 1, - "enum": [ - "US", - "EU", - "asia-east1", - "asia-east2", - "asia-northeast1", - "asia-northeast2", - "asia-northeast3", - "asia-south1", - "asia-south2", - "asia-southeast1", - "asia-southeast2", - "australia-southeast1", - "australia-southeast2", - "europe-central2", - "europe-north1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "europe-west6", - "northamerica-northeast1", - "northamerica-northeast2", - "southamerica-east1", - "southamerica-west1", - "us-central1", - "us-east1", - "us-east4", - "us-west1", - "us-west2", - "us-west3", - "us-west4" - ] - }, - "dataset_id": { - "type": "string", - "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", - "title": "Default Dataset ID", - "order": 2 - }, - "loading_method": { - "type": "object", - "title": "Loading Method", - "description": "Loading method used to send select the way data will be uploaded to BigQuery.
Standard Inserts - Direct uploading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In almost all cases, you should use staging.
GCS Staging - Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO table to upload the file. Recommended for most workloads for better speed and scalability. Read more about GCS Staging here.", - "order": 3, - "oneOf": [ - { - "title": "Standard Inserts", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "GCS Staging", - "required": [ - "method", - "gcs_bucket_name", - "gcs_bucket_path", - "credential" - ], - "properties": { - "method": { - "type": "string", - "const": "GCS Staging", - "order": 0 - }, - "credential": { - "title": "Credential", - "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", - "type": "object", - "order": 1, - "oneOf": [ - { - "title": "HMAC key", - "required": [ - "credential_type", - "hmac_key_access_id", - "hmac_key_secret" - ], - "properties": { - "credential_type": { - "type": "string", - "const": "HMAC_KEY", - "order": 0 - }, - "hmac_key_access_id": { - "type": "string", - "description": "HMAC key access ID. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long.", - "title": "HMAC Key Access ID", - "airbyte_secret": true, - "examples": ["1234567890abcdefghij1234"], - "order": 1 - }, - "hmac_key_secret": { - "type": "string", - "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string.", - "title": "HMAC Key Secret", - "airbyte_secret": true, - "examples": [ - "1234567890abcdefghij1234567890ABCDEFGHIJ" - ], - "order": 2 - } - } - } - ] - }, - "gcs_bucket_name": { - "title": "GCS Bucket Name", - "type": "string", - "description": "The name of the GCS bucket. Read more here.", - "examples": ["airbyte_sync"], - "order": 2 - }, - "gcs_bucket_path": { - "title": "GCS Bucket Path", - "description": "Directory under the GCS bucket where data will be written.", - "type": "string", - "examples": ["data_sync/test"], - "order": 3 - }, - "keep_files_in_gcs-bucket": { - "type": "string", - "description": "This upload method is supposed to temporary store records in GCS bucket. By this select you can chose if these records should be removed from GCS when migration has finished. The default \"Delete all tmp files from GCS\" value is used if not set explicitly.", - "title": "GCS Tmp Files Afterward Processing (Optional)", - "default": "Delete all tmp files from GCS", - "enum": [ - "Delete all tmp files from GCS", - "Keep all tmp files in GCS" - ], - "order": 4 - } - } - } - ] - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", - "title": "Service Account Key JSON (Required for cloud, optional for open-source)", - "airbyte_secret": true, - "order": 4 - }, - "transformation_priority": { - "type": "string", - "description": "Interactive run type means that the query is executed as soon as possible, and these queries count towards concurrent rate limit and daily limit. Read more about interactive run type here. Batch queries are queued and started as soon as idle resources are available in the BigQuery shared resource pool, which usually occurs within a few minutes. Batch queries don’t count towards your concurrent rate limit. Read more about batch queries here. The default \"interactive\" value is used if not set explicitly.", - "title": "Transformation Query Run Type (Optional)", - "default": "interactive", - "enum": ["interactive", "batch"], - "order": 5 - }, - "big_query_client_buffer_size_mb": { - "title": "Google BigQuery Client Chunk Size (Optional)", - "description": "Google BigQuery client's chunk (buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MB value is used if not set explicitly. Read more here.", - "type": "integer", - "minimum": 1, - "maximum": 15, - "default": 15, - "examples": ["15"], - "order": 6 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "079d5540-f236-4294-ba7c-ade8fd918496", - "name": "BigQuery (denormalized typed struct)", - "dockerRepository": "airbyte/destination-bigquery-denormalized", - "dockerImageTag": "1.1.15", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "icon": "bigquery.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Denormalized Typed Struct Destination Spec", - "type": "object", - "required": ["project_id", "dataset_id"], - "additionalProperties": true, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", - "title": "Project ID", - "order": 0 - }, - "dataset_id": { - "type": "string", - "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", - "title": "Default Dataset ID", - "order": 1 - }, - "loading_method": { - "type": "object", - "title": "Loading Method *", - "description": "Loading method used to send select the way data will be uploaded to BigQuery.
Standard Inserts - Direct uploading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In almost all cases, you should use staging.
GCS Staging - Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO table to upload the file. Recommended for most workloads for better speed and scalability. Read more about GCS Staging here.", - "order": 2, - "oneOf": [ - { - "title": "Standard Inserts", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "GCS Staging", - "type": "object", - "required": [ - "method", - "gcs_bucket_name", - "gcs_bucket_path", - "credential" - ], - "properties": { - "method": { - "type": "string", - "const": "GCS Staging", - "order": 0 - }, - "credential": { - "title": "Credential", - "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", - "type": "object", - "order": 1, - "oneOf": [ - { - "title": "HMAC key", - "order": 0, - "required": [ - "credential_type", - "hmac_key_access_id", - "hmac_key_secret" - ], - "properties": { - "credential_type": { - "type": "string", - "const": "HMAC_KEY", - "order": 0 - }, - "hmac_key_access_id": { - "type": "string", - "description": "HMAC key access ID. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long.", - "title": "HMAC Key Access ID", - "airbyte_secret": true, - "examples": ["1234567890abcdefghij1234"], - "order": 1 - }, - "hmac_key_secret": { - "type": "string", - "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string.", - "title": "HMAC Key Secret", - "airbyte_secret": true, - "examples": [ - "1234567890abcdefghij1234567890ABCDEFGHIJ" - ], - "order": 2 - } - } - } - ] - }, - "gcs_bucket_name": { - "title": "GCS Bucket Name", - "type": "string", - "description": "The name of the GCS bucket. Read more here.", - "examples": ["airbyte_sync"], - "order": 2 - }, - "gcs_bucket_path": { - "title": "GCS Bucket Path", - "description": "Directory under the GCS bucket where data will be written. Read more here.", - "type": "string", - "examples": ["data_sync/test"], - "order": 3 - }, - "keep_files_in_gcs-bucket": { - "type": "string", - "description": "This upload method is supposed to temporary store records in GCS bucket. By this select you can chose if these records should be removed from GCS when migration has finished. The default \"Delete all tmp files from GCS\" value is used if not set explicitly.", - "title": "GCS Tmp Files Afterward Processing (Optional)", - "default": "Delete all tmp files from GCS", - "enum": [ - "Delete all tmp files from GCS", - "Keep all tmp files in GCS" - ], - "order": 4 - } - } - } - ] - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", - "title": "Service Account Key JSON (Required for cloud, optional for open-source)", - "airbyte_secret": true, - "order": 3 - }, - "dataset_location": { - "type": "string", - "description": "The location of the dataset. Warning: Changes made after creation will not be applied. The default \"US\" value is used if not set explicitly. Read more here.", - "title": "Dataset Location (Optional)", - "default": "US", - "order": 4, - "enum": [ - "US", - "EU", - "asia-east1", - "asia-east2", - "asia-northeast1", - "asia-northeast2", - "asia-northeast3", - "asia-south1", - "asia-south2", - "asia-southeast1", - "asia-southeast2", - "australia-southeast1", - "australia-southeast2", - "europe-central2", - "europe-north1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "europe-west6", - "northamerica-northeast1", - "northamerica-northeast2", - "southamerica-east1", - "southamerica-west1", - "us-central1", - "us-east1", - "us-east4", - "us-west1", - "us-west2", - "us-west3", - "us-west4" - ] - }, - "big_query_client_buffer_size_mb": { - "title": "Google BigQuery Client Chunk Size (Optional)", - "description": "Google BigQuery client's chunk (buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MB value is used if not set explicitly. Read more here.", - "type": "integer", - "minimum": 1, - "maximum": 15, - "default": 15, - "examples": ["15"], - "order": 5 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "beta", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "707456df-6f4f-4ced-b5c6-03f73bcad1c5", - "name": "Cassandra", - "dockerRepository": "airbyte/destination-cassandra", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/cassandra", - "icon": "cassandra.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/cassandra", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Cassandra Destination Spec", - "type": "object", - "required": ["keyspace", "username", "password", "address", "port"], - "additionalProperties": true, - "properties": { - "keyspace": { - "title": "Keyspace", - "description": "Default Cassandra keyspace to create data in.", - "type": "string", - "order": 0 - }, - "username": { - "title": "Username", - "description": "Username to use to access Cassandra.", - "type": "string", - "order": 1 - }, - "password": { - "title": "Password", - "description": "Password associated with Cassandra.", - "type": "string", - "airbyte_secret": true, - "order": 2 - }, - "address": { - "title": "Address", - "description": "Address to connect to.", - "type": "string", - "examples": ["localhost,127.0.0.1"], - "order": 3 - }, - "port": { - "title": "Port", - "description": "Port of Cassandra.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9042, - "order": 4 - }, - "datacenter": { - "title": "Datacenter", - "description": "Datacenter of the cassandra cluster.", - "type": "string", - "default": "datacenter1", - "order": 5 - }, - "replication": { - "title": "Replication factor", - "type": "integer", - "description": "Indicates to how many nodes the data should be replicated to.", - "default": 1, - "order": 6 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "81740ce8-d764-4ea7-94df-16bb41de36ae", - "name": "Chargify (Keen)", - "dockerRepository": "airbyte/destination-keen", - "dockerImageTag": "0.2.4", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/keen", - "icon": "chargify.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/keen", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Keen Spec", - "type": "object", - "required": ["project_id", "api_key"], - "additionalProperties": false, - "properties": { - "project_id": { - "description": "To get Keen Project ID, navigate to the Access tab from the left-hand, side panel and check the Project Details section.", - "title": "Project ID", - "type": "string", - "examples": ["58b4acc22ba938934e888322e"] - }, - "api_key": { - "title": "API Key", - "description": "To get Keen Master API Key, navigate to the Access tab from the left-hand, side panel and check the Project Details section.", - "type": "string", - "examples": ["ABCDEFGHIJKLMNOPRSTUWXYZ"], - "airbyte_secret": true - }, - "infer_timestamp": { - "title": "Infer Timestamp", - "description": "Allow connector to guess keen.timestamp value based on the streamed data.", - "type": "boolean", - "default": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "ce0d828e-1dc4-496c-b122-2da42e637e48", - "name": "Clickhouse", - "dockerRepository": "airbyte/destination-clickhouse", - "dockerImageTag": "0.1.11", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ClickHouse Destination Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "HTTP port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 8123, - "examples": ["8123"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 3 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "ssl": { - "title": "SSL Connection", - "description": "Encrypt data using SSL.", - "type": "boolean", - "default": false, - "order": 6 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "072d5540-f236-4294-ba7c-ade8fd918496", - "name": "Databricks Lakehouse", - "dockerRepository": "airbyte/destination-databricks", - "dockerImageTag": "0.2.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/databricks", - "icon": "databricks.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/databricks", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Databricks Lakehouse Destination Spec", - "type": "object", - "required": [ - "accept_terms", - "databricks_server_hostname", - "databricks_http_path", - "databricks_personal_access_token", - "data_source" - ], - "properties": { - "accept_terms": { - "title": "Agree to the Databricks JDBC Driver Terms & Conditions", - "type": "boolean", - "description": "You must agree to the Databricks JDBC Driver Terms & Conditions to use this connector.", - "default": false, - "order": 1 - }, - "databricks_server_hostname": { - "title": "Server Hostname", - "type": "string", - "description": "Databricks Cluster Server Hostname.", - "examples": ["abc-12345678-wxyz.cloud.databricks.com"], - "order": 2 - }, - "databricks_http_path": { - "title": "HTTP Path", - "type": "string", - "description": "Databricks Cluster HTTP Path.", - "examples": ["sql/protocolvx/o/1234567489/0000-1111111-abcd90"], - "order": 3 - }, - "databricks_port": { - "title": "Port", - "type": "string", - "description": "Databricks Cluster Port.", - "default": "443", - "examples": ["443"], - "order": 4 - }, - "databricks_personal_access_token": { - "title": "Access Token", - "type": "string", - "description": "Databricks Personal Access Token for making authenticated requests.", - "examples": ["dapi0123456789abcdefghij0123456789AB"], - "airbyte_secret": true, - "order": 5 - }, - "database_schema": { - "title": "Database Schema", - "type": "string", - "description": "The default schema tables are written to if the source does not specify a namespace. Unless specifically configured, the usual value for this field is \"public\".", - "default": "public", - "examples": ["public"], - "order": 6 - }, - "data_source": { - "title": "Data Source", - "type": "object", - "description": "Storage on which the delta lake is built.", - "oneOf": [ - { - "title": "Amazon S3", - "required": [ - "data_source_type", - "s3_bucket_name", - "s3_bucket_path", - "s3_bucket_region", - "s3_access_key_id", - "s3_secret_access_key" - ], - "properties": { - "data_source_type": { - "type": "string", - "enum": ["S3"], - "default": "S3", - "order": 1 - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "The name of the S3 bucket to use for intermittent staging of the data.", - "examples": ["airbyte.staging"], - "order": 2 - }, - "s3_bucket_path": { - "title": "S3 Bucket Path", - "type": "string", - "description": "The directory under the S3 bucket where data will be written.", - "examples": ["data_sync/test"], - "order": 3 - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "The region of the S3 staging bucket to use if utilising a copy strategy.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 4 - }, - "s3_access_key_id": { - "type": "string", - "description": "The Access Key Id granting allow one to access the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket.", - "title": "S3 Access Key ID", - "examples": ["A012345678910EXAMPLE"], - "airbyte_secret": true, - "order": 5 - }, - "s3_secret_access_key": { - "title": "S3 Secret Access Key", - "type": "string", - "description": "The corresponding secret to the above access key id.", - "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"], - "airbyte_secret": true, - "order": 6 - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 7 - } - } - } - ], - "order": 7 - }, - "purge_staging_data": { - "title": "Purge Staging Files and Tables", - "type": "boolean", - "description": "Default to 'true'. Switch it to 'false' for debugging purpose.", - "default": true, - "order": 8 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "8ccd8909-4e99-4141-b48d-4984b70b2d89", - "name": "DynamoDB", - "dockerRepository": "airbyte/destination-dynamodb", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/dynamodb", - "icon": "dynamodb.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/dynamodb", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "DynamoDB Destination Spec", - "type": "object", - "required": [ - "dynamodb_table_name_prefix", - "dynamodb_region", - "access_key_id", - "secret_access_key" - ], - "additionalProperties": false, - "properties": { - "dynamodb_endpoint": { - "title": "Endpoint", - "type": "string", - "default": "", - "description": "This is your DynamoDB endpoint url.(if you are working with AWS DynamoDB, just leave empty).", - "examples": ["http://localhost:9000"] - }, - "dynamodb_table_name_prefix": { - "title": "Table name prefix", - "type": "string", - "description": "The prefix to use when naming DynamoDB tables.", - "examples": ["airbyte_sync"] - }, - "dynamodb_region": { - "title": "DynamoDB Region", - "type": "string", - "default": "", - "description": "The region of the DynamoDB.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ] - }, - "access_key_id": { - "type": "string", - "description": "The access key id to access the DynamoDB. Airbyte requires Read and Write permissions to the DynamoDB.", - "title": "DynamoDB Key Id", - "airbyte_secret": true, - "examples": ["A012345678910EXAMPLE"] - }, - "secret_access_key": { - "type": "string", - "description": "The corresponding secret to the access key id.", - "title": "DynamoDB Access Key", - "airbyte_secret": true, - "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "68f351a7-2745-4bef-ad7f-996b8e51bb8c", - "name": "ElasticSearch", - "dockerRepository": "airbyte/destination-elasticsearch", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/elasticsearch", - "icon": "elasticsearch.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/elasticsearch", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Elasticsearch Connection Configuration", - "type": "object", - "required": ["endpoint"], - "additionalProperties": false, - "properties": { - "endpoint": { - "title": "Server Endpoint", - "type": "string", - "description": "The full url of the Elasticsearch server" - }, - "upsert": { - "type": "boolean", - "title": "Upsert Records", - "description": "If a primary key identifier is defined in the source, an upsert will be performed using the primary key value as the elasticsearch doc id. Does not support composite primary keys.", - "default": true - }, - "authenticationMethod": { - "title": "Authentication Method", - "type": "object", - "description": "The type of authentication to be used", - "oneOf": [ - { - "title": "None", - "additionalProperties": false, - "description": "No authentication will be used", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "none" - } - } - }, - { - "title": "Api Key/Secret", - "additionalProperties": false, - "description": "Use a api key and secret combination to authenticate", - "required": ["method", "apiKeyId", "apiKeySecret"], - "properties": { - "method": { - "type": "string", - "const": "secret" - }, - "apiKeyId": { - "title": "API Key ID", - "description": "The Key ID to used when accessing an enterprise Elasticsearch instance.", - "type": "string" - }, - "apiKeySecret": { - "title": "API Key Secret", - "description": "The secret associated with the API Key ID.", - "type": "string", - "airbyte_secret": true - } - } - }, - { - "title": "Username/Password", - "additionalProperties": false, - "description": "Basic auth header with a username and password", - "required": ["method", "username", "password"], - "properties": { - "method": { - "type": "string", - "const": "basic" - }, - "username": { - "title": "Username", - "description": "Basic auth username to access a secure Elasticsearch server", - "type": "string" - }, - "password": { - "title": "Password", - "description": "Basic auth password to access a secure Elasticsearch server", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"], - "supportsNamespaces": true - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "a7bcc9d8-13b3-4e49-b80d-d020b90045e3", - "name": "End-to-End Testing (/dev/null)", - "dockerRepository": "airbyte/destination-dev-null", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/e2e-test", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/e2e-test", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "E2E Test (/dev/null) Destination Spec", - "type": "object", - "oneOf": [ - { - "title": "Silent", - "required": ["type"], - "properties": { - "type": { - "type": "string", - "const": "SILENT", - "default": "SILENT" - } - } - } - ] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "18081484-02a5-4662-8dba-b270b582f321", - "name": "Firebolt", - "dockerRepository": "airbyte/destination-firebolt", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firebolt", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firebolt", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Firebolt Spec", - "type": "object", - "required": ["username", "password", "database"], - "additionalProperties": false, - "properties": { - "username": { - "type": "string", - "title": "Username", - "description": "Firebolt email address you use to login.", - "examples": ["username@email.com"], - "order": 0 - }, - "password": { - "type": "string", - "title": "Password", - "description": "Firebolt password.", - "airbyte_secret": true, - "order": 1 - }, - "account": { - "type": "string", - "title": "Account", - "description": "Firebolt account to login." - }, - "host": { - "type": "string", - "title": "Host", - "description": "The host name of your Firebolt database.", - "examples": ["api.app.firebolt.io"] - }, - "database": { - "type": "string", - "title": "Database", - "description": "The database to connect to." - }, - "engine": { - "type": "string", - "title": "Engine", - "description": "Engine name or url to connect to." - }, - "loading_method": { - "type": "object", - "title": "Loading Method", - "description": "Loading method used to select the way data will be uploaded to Firebolt", - "oneOf": [ - { - "title": "SQL Inserts", - "additionalProperties": false, - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "SQL" - } - } - }, - { - "title": "External Table via S3", - "additionalProperties": false, - "required": [ - "method", - "s3_bucket", - "s3_region", - "aws_key_id", - "aws_key_secret" - ], - "properties": { - "method": { - "type": "string", - "const": "S3" - }, - "s3_bucket": { - "type": "string", - "title": "S3 bucket name", - "description": "The name of the S3 bucket." - }, - "s3_region": { - "type": "string", - "title": "S3 region name", - "description": "Region name of the S3 bucket.", - "examples": ["us-east-1"] - }, - "aws_key_id": { - "type": "string", - "title": "AWS Key ID", - "airbyte_secret": true, - "description": "AWS access key granting read and write access to S3." - }, - "aws_key_secret": { - "type": "string", - "title": "AWS Key Secret", - "airbyte_secret": true, - "description": "Corresponding secret part of the AWS Key" - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "ca8f6566-e555-4b40-943a-545bf123117a", - "name": "Google Cloud Storage (GCS)", - "dockerRepository": "airbyte/destination-gcs", - "dockerImageTag": "0.2.10", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/gcs", - "icon": "googlecloudstorage.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/gcs", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "GCS Destination Spec", - "type": "object", - "required": [ - "gcs_bucket_name", - "gcs_bucket_path", - "credential", - "format" - ], - "properties": { - "gcs_bucket_name": { - "title": "GCS Bucket Name", - "order": 1, - "type": "string", - "description": "You can find the bucket name in the App Engine Admin console Application Settings page, under the label Google Cloud Storage Bucket. Read more here.", - "examples": ["airbyte_sync"] - }, - "gcs_bucket_path": { - "title": "GCS Bucket Path", - "description": "GCS Bucket Path string Subdirectory under the above bucket to sync the data into.", - "order": 2, - "type": "string", - "examples": ["data_sync/test"] - }, - "gcs_bucket_region": { - "title": "GCS Bucket Region (Optional)", - "type": "string", - "order": 3, - "default": "us", - "description": "Select a Region of the GCS Bucket. Read more here.", - "enum": [ - "northamerica-northeast1", - "northamerica-northeast2", - "us-central1", - "us-east1", - "us-east4", - "us-west1", - "us-west2", - "us-west3", - "us-west4", - "southamerica-east1", - "southamerica-west1", - "europe-central2", - "europe-north1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "europe-west6", - "asia-east1", - "asia-east2", - "asia-northeast1", - "asia-northeast2", - "asia-northeast3", - "asia-south1", - "asia-south2", - "asia-southeast1", - "asia-southeast2", - "australia-southeast1", - "australia-southeast2", - "asia", - "eu", - "us", - "asia1", - "eur4", - "nam4" - ] - }, - "credential": { - "title": "Authentication", - "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", - "type": "object", - "order": 0, - "oneOf": [ - { - "title": "HMAC Key", - "required": [ - "credential_type", - "hmac_key_access_id", - "hmac_key_secret" - ], - "properties": { - "credential_type": { - "type": "string", - "enum": ["HMAC_KEY"], - "default": "HMAC_KEY" - }, - "hmac_key_access_id": { - "type": "string", - "description": "When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long. Read more here.", - "title": "Access ID", - "airbyte_secret": true, - "order": 0, - "examples": ["1234567890abcdefghij1234"] - }, - "hmac_key_secret": { - "type": "string", - "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string. Read more here.", - "title": "Secret", - "airbyte_secret": true, - "order": 1, - "examples": ["1234567890abcdefghij1234567890ABCDEFGHIJ"] - } - } - } - ] - }, - "format": { - "title": "Output Format", - "type": "object", - "description": "Output data format. One of the following formats must be selected - AVRO format, PARQUET format, CSV format, or JSONL format.", - "order": 4, - "oneOf": [ - { - "title": "Avro: Apache Avro", - "required": ["format_type", "compression_codec"], - "properties": { - "format_type": { - "type": "string", - "enum": ["Avro"], - "default": "Avro" - }, - "compression_codec": { - "title": "Compression Codec", - "description": "The compression algorithm used to compress data. Default to no compression.", - "type": "object", - "oneOf": [ - { - "title": "No Compression", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["no compression"], - "default": "no compression" - } - } - }, - { - "title": "Deflate", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["Deflate"], - "default": "Deflate" - }, - "compression_level": { - "title": "Deflate level (Optional)", - "description": "0: no compression & fastest, 9: best compression & slowest.", - "type": "integer", - "default": 0, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "bzip2", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["bzip2"], - "default": "bzip2" - } - } - }, - { - "title": "xz", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["xz"], - "default": "xz" - }, - "compression_level": { - "title": "Compression Level (Optional)", - "description": "The presets 0-3 are fast presets with medium compression. The presets 4-6 are fairly slow presets with high compression. The default preset is 6. The presets 7-9 are like the preset 6 but use bigger dictionaries and have higher compressor and decompressor memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively. Read more here for details.", - "type": "integer", - "default": 6, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "zstandard", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["zstandard"], - "default": "zstandard" - }, - "compression_level": { - "title": "Compression Level (Optional)", - "description": "Negative levels are 'fast' modes akin to lz4 or snappy, levels above 9 are generally for archival purposes, and levels above 18 use a lot of memory.", - "type": "integer", - "default": 3, - "minimum": -5, - "maximum": 22 - }, - "include_checksum": { - "title": "Include Checksum (Optional)", - "description": "If true, include a checksum with each data block.", - "type": "boolean", - "default": false - } - } - }, - { - "title": "snappy", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["snappy"], - "default": "snappy" - } - } - } - ] - } - } - }, - { - "title": "CSV: Comma-Separated Values", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "enum": ["CSV"], - "default": "CSV" - }, - "flattening": { - "type": "string", - "title": "Normalization (Optional)", - "description": "Whether the input JSON data should be normalized (flattened) in the output CSV. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "JSON Lines: newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "enum": ["JSONL"], - "default": "JSONL" - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "Parquet: Columnar Storage", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "enum": ["Parquet"], - "default": "Parquet" - }, - "compression_codec": { - "title": "Compression Codec (Optional)", - "description": "The compression algorithm used to compress data pages.", - "type": "string", - "default": "UNCOMPRESSED", - "enum": [ - "UNCOMPRESSED", - "SNAPPY", - "GZIP", - "LZO", - "BROTLI", - "LZ4", - "ZSTD" - ] - }, - "block_size_mb": { - "title": "Block Size (Row Group Size) (MB) (Optional)", - "description": "This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. Default: 128 MB.", - "type": "integer", - "default": 128, - "examples": [128] - }, - "max_padding_size_mb": { - "title": "Max Padding Size (MB) (Optional)", - "description": "Maximum size allowed as padding to align row groups. This is also the minimum size of a row group. Default: 8 MB.", - "type": "integer", - "default": 8, - "examples": [8] - }, - "page_size_kb": { - "title": "Page Size (KB) (Optional)", - "description": "The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_page_size_kb": { - "title": "Dictionary Page Size (KB) (Optional)", - "description": "There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_encoding": { - "title": "Dictionary Encoding (Optional)", - "description": "Default: true.", - "type": "boolean", - "default": true - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"], - "$schema": "http://json-schema.org/draft-07/schema#" - }, - "public": true, - "custom": false, - "releaseStage": "beta", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "27dc7500-6d1b-40b1-8b07-e2f2aea3c9f4", - "name": "Google Firestore", - "dockerRepository": "airbyte/destination-firestore", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firestore", - "icon": "firestore.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firestore", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Google Firestore", - "type": "object", - "required": ["project_id"], - "additionalProperties": false, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset.", - "title": "Project ID" - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", - "title": "Credentials JSON", - "airbyte_secret": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append", "overwrite"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "356668e2-7e34-47f3-a3b0-67a8a481b692", - "name": "Google PubSub", - "dockerRepository": "airbyte/destination-pubsub", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pubsub", - "icon": "googlepubsub.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pubsub", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google PubSub Destination Spec", - "type": "object", - "required": ["project_id", "topic_id", "credentials_json"], - "additionalProperties": true, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target PubSub.", - "title": "Project ID" - }, - "topic_id": { - "type": "string", - "description": "The PubSub topic ID in the given GCP project ID.", - "title": "PubSub Topic ID" - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key.", - "title": "Credentials JSON", - "airbyte_secret": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a", - "name": "Google Sheets", - "dockerRepository": "airbyte/destination-google-sheets", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/google-sheets", - "icon": "google-sheets.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/google-sheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Google Sheets", - "type": "object", - "required": ["spreadsheet_id", "credentials"], - "additionalProperties": false, - "properties": { - "spreadsheet_id": { - "type": "string", - "title": "Spreadsheet Link", - "description": "The link to your spreadsheet. See this guide for more details.", - "examples": [ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG/edit" - ] - }, - "credentials": { - "type": "object", - "title": "* Authentication via Google (OAuth)", - "description": "Google API Credentials for connecting to Google Sheets and Google Drive APIs", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Google Sheets developer application.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Google Sheets developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining new access token.", - "airbyte_secret": true - } - } - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "d4353156-9217-4cad-8dd7-c108fd4f74cf", - "name": "MS SQL Server", - "dockerRepository": "airbyte/destination-mssql-strict-encrypt", - "dockerImageTag": "0.1.20", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mssql", - "icon": "mssql.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mssql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MS SQL Server Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database", "schema"], - "properties": { - "host": { - "title": "Host", - "description": "The host name of the MSSQL database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "The port of the MSSQL database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 1433, - "examples": ["1433"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "The name of the MSSQL database.", - "type": "string", - "order": 2 - }, - "schema": { - "title": "Default Schema", - "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", - "type": "string", - "examples": ["public"], - "default": "public", - "order": 3 - }, - "username": { - "title": "User", - "description": "The username which is used to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "The password associated with this username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 6 - }, - "ssl_method": { - "title": "SSL Method", - "type": "object", - "description": "The encryption method which is used to communicate with the database.", - "order": 7, - "oneOf": [ - { - "title": "Encrypted (trust server certificate)", - "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", - "required": ["ssl_method"], - "type": "object", - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_trust_server_certificate", - "enum": ["encrypted_trust_server_certificate"], - "default": "encrypted_trust_server_certificate" - } - } - }, - { - "title": "Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": [ - "ssl_method", - "trustStoreName", - "trustStorePassword" - ], - "type": "object", - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" - }, - "hostNameInCertificate": { - "title": "Host Name In Certificate", - "type": "string", - "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 8 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "294a4790-429b-40ae-9516-49826b9702e1", - "name": "MariaDB ColumnStore", - "dockerRepository": "airbyte/destination-mariadb-columnstore", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore", - "icon": "mariadb.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MariaDB Columnstore Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "The Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "The Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "title": "Database", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "username": { - "title": "Username", - "description": "The Username which is used to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "The Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", - "name": "MeiliSearch", - "dockerRepository": "airbyte/destination-meilisearch", - "dockerImageTag": "0.2.13", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch", - "icon": "meilisearch.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MeiliSearch Destination Spec", - "type": "object", - "required": ["host"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the MeiliSearch instance.", - "type": "string", - "order": 0 - }, - "api_key": { - "title": "API Key", - "airbyte_secret": true, - "description": "MeiliSearch API Key. See the docs for more information on how to obtain this key.", - "type": "string", - "order": 1 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "8b746512-8c2e-6ac1-4adc-b59faafd473c", - "name": "MongoDB", - "dockerRepository": "airbyte/destination-mongodb-strict-encrypt", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mongodb", - "icon": "mongodb.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mongodb", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MongoDB Destination Spec", - "type": "object", - "required": ["database", "auth_type"], - "additionalProperties": true, - "properties": { - "instance_type": { - "description": "MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", - "title": "MongoDb Instance Type", - "type": "object", - "order": 0, - "oneOf": [ - { - "title": "Standalone MongoDb Instance", - "required": ["instance", "host", "port"], - "properties": { - "instance": { - "type": "string", - "enum": ["standalone"], - "default": "standalone" - }, - "host": { - "title": "Host", - "type": "string", - "description": "The Host of a Mongo database to be replicated.", - "order": 0 - }, - "port": { - "title": "Port", - "type": "integer", - "description": "The Port of a Mongo database to be replicated.", - "minimum": 0, - "maximum": 65536, - "default": 27017, - "examples": ["27017"], - "order": 1 - } - } - }, - { - "title": "Replica Set", - "required": ["instance", "server_addresses"], - "properties": { - "instance": { - "type": "string", - "enum": ["replica"], - "default": "replica" - }, - "server_addresses": { - "title": "Server addresses", - "type": "string", - "description": "The members of a replica set. Please specify `host`:`port` of each member seperated by comma.", - "examples": ["host1:27017,host2:27017,host3:27017"], - "order": 0 - }, - "replica_set": { - "title": "Replica Set", - "type": "string", - "description": "A replica set name.", - "order": 1 - } - } - }, - { - "title": "MongoDB Atlas", - "additionalProperties": false, - "required": ["instance", "cluster_url"], - "properties": { - "instance": { - "type": "string", - "enum": ["atlas"], - "default": "atlas" - }, - "cluster_url": { - "title": "Cluster URL", - "type": "string", - "description": "URL of a cluster to connect to.", - "order": 0 - } - } - } - ] - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "auth_type": { - "title": "Authorization type", - "type": "object", - "description": "Authorization type.", - "oneOf": [ - { - "title": "None", - "additionalProperties": false, - "description": "None.", - "required": ["authorization"], - "type": "object", - "properties": { - "authorization": { - "type": "string", - "const": "none" - } - } - }, - { - "title": "Login/Password", - "additionalProperties": false, - "description": "Login/Password.", - "required": ["authorization", "username", "password"], - "type": "object", - "properties": { - "authorization": { - "type": "string", - "const": "login/password" - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 1 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 2 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "ca81ee7c-3163-4246-af40-094cc31e5e42", - "name": "MySQL", - "dockerRepository": "airbyte/destination-mysql-strict-encrypt", - "dockerImageTag": "0.1.20", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mysql", - "icon": "mysql.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mysql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MySQL Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 6 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72", - "name": "Oracle", - "dockerRepository": "airbyte/destination-oracle-strict-encrypt", - "dockerImageTag": "0.1.19", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", - "icon": "oracle.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Oracle Destination Spec", - "type": "object", - "required": ["host", "port", "username", "sid"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "The hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "The port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 1521, - "examples": ["1521"], - "order": 1 - }, - "sid": { - "title": "SID", - "description": "The System Identifier uniquely distinguishes the instance from any other instance on the same computer.", - "type": "string", - "order": 2 - }, - "username": { - "title": "User", - "description": "The username to access the database. This user must have CREATE USER privileges in the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "The password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 5 - }, - "schema": { - "title": "Default Schema", - "description": "The default schema is used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. The usual value for this field is \"airbyte\". In Oracle, schemas and users are the same thing, so the \"user\" parameter is used as the login credentials and this is used for the default Airbyte message schema.", - "type": "string", - "examples": ["airbyte"], - "default": "airbyte", - "order": 6 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", - "name": "Postgres", - "dockerRepository": "airbyte/destination-postgres-strict-encrypt", - "dockerImageTag": "0.3.22", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", - "icon": "postgresql.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database", "schema"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5432, - "examples": ["5432"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "schema": { - "title": "Default Schema", - "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", - "type": "string", - "examples": ["public"], - "default": "public", - "order": 3 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "ssl_mode": { - "title": "SSL modes", - "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the source database\n prefer - Chose this mode to allow unencrypted connection only if the source database does not support encryption\n require - Chose this mode to always require encryption. If the source database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the source database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the source database server\n See more information - in the docs.", - "type": "object", - "order": 7, - "oneOf": [ - { - "title": "prefer", - "additionalProperties": false, - "description": "Prefer SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "prefer", - "enum": ["prefer"], - "default": "prefer", - "order": 0 - } - } - }, - { - "title": "require", - "additionalProperties": false, - "description": "Require SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "require", - "enum": ["require"], - "default": "require", - "order": 0 - } - } - }, - { - "title": "verify-ca", - "additionalProperties": false, - "description": "Verify-ca SSL mode.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify-ca", - "enum": ["verify-ca"], - "default": "verify-ca", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_key_password": { - "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "verify-full", - "additionalProperties": false, - "description": "Verify-full SSL mode.", - "required": [ - "mode", - "ca_certificate", - "client_certificate", - "client_key" - ], - "properties": { - "mode": { - "type": "string", - "const": "verify-full", - "enum": ["verify-full"], - "default": "verify-full", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client certificate", - "description": "Client certificate", - "airbyte_secret": true, - "multiline": true, - "order": 2 - }, - "client_key": { - "type": "string", - "title": "Client key", - "description": "Client key", - "airbyte_secret": true, - "multiline": true, - "order": 3 - }, - "client_key_password": { - "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 8 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "2340cbba-358e-11ec-8d3d-0242ac130203", - "name": "Pulsar", - "dockerRepository": "airbyte/destination-pulsar", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pulsar", - "icon": "pulsar.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pulsar", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pulsar Destination Spec", - "type": "object", - "required": [ - "brokers", - "use_tls", - "topic_type", - "topic_tenant", - "topic_namespace", - "topic_pattern", - "compression_type", - "send_timeout_ms", - "max_pending_messages", - "max_pending_messages_across_partitions", - "batching_enabled", - "batching_max_messages", - "batching_max_publish_delay", - "block_if_queue_full" - ], - "additionalProperties": true, - "properties": { - "brokers": { - "title": "Pulsar brokers", - "description": "A list of host/port pairs to use for establishing the initial connection to the Pulsar cluster.", - "type": "string", - "examples": ["broker1:6650,broker2:6650"] - }, - "use_tls": { - "title": "Use TLS", - "description": "Whether to use TLS encryption on the connection.", - "type": "boolean", - "default": false - }, - "topic_type": { - "title": "Topic type", - "description": "It identifies type of topic. Pulsar supports two kind of topics: persistent and non-persistent. In persistent topic, all messages are durably persisted on disk (that means on multiple disks unless the broker is standalone), whereas non-persistent topic does not persist message into storage disk.", - "type": "string", - "default": "persistent", - "enum": ["persistent", "non-persistent"] - }, - "topic_tenant": { - "title": "Topic tenant", - "description": "The topic tenant within the instance. Tenants are essential to multi-tenancy in Pulsar, and spread across clusters.", - "type": "string", - "default": "public", - "examples": ["public"] - }, - "topic_namespace": { - "title": "Topic namespace", - "description": "The administrative unit of the topic, which acts as a grouping mechanism for related topics. Most topic configuration is performed at the namespace level. Each tenant has one or multiple namespaces.", - "type": "string", - "default": "default", - "examples": ["default"] - }, - "topic_pattern": { - "title": "Topic pattern", - "description": "Topic pattern in which the records will be sent. You can use patterns like '{namespace}' and/or '{stream}' to send the message to a specific topic based on these values. Notice that the topic name will be transformed to a standard naming convention.", - "type": "string", - "examples": ["sample.topic", "{namespace}.{stream}.sample"] - }, - "topic_test": { - "title": "Test topic", - "description": "Topic to test if Airbyte can produce messages.", - "type": "string", - "examples": ["test.topic"] - }, - "producer_name": { - "title": "Producer name", - "description": "Name for the producer. If not filled, the system will generate a globally unique name which can be accessed with.", - "type": "string", - "examples": ["airbyte-producer"] - }, - "producer_sync": { - "title": "Sync producer", - "description": "Wait synchronously until the record has been sent to Pulsar.", - "type": "boolean", - "default": false - }, - "compression_type": { - "title": "Compression type", - "description": "Compression type for the producer.", - "type": "string", - "default": "NONE", - "enum": ["NONE", "LZ4", "ZLIB", "ZSTD", "SNAPPY"] - }, - "send_timeout_ms": { - "title": "Message send timeout", - "description": "If a message is not acknowledged by a server before the send-timeout expires, an error occurs (in ms).", - "type": "integer", - "default": 30000 - }, - "max_pending_messages": { - "title": "Max pending messages", - "description": "The maximum size of a queue holding pending messages.", - "type": "integer", - "default": 1000 - }, - "max_pending_messages_across_partitions": { - "title": "Max pending messages across partitions", - "description": "The maximum number of pending messages across partitions.", - "type": "integer", - "default": 50000 - }, - "batching_enabled": { - "title": "Enable batching", - "description": "Control whether automatic batching of messages is enabled for the producer.", - "type": "boolean", - "default": true - }, - "batching_max_messages": { - "title": "Batching max messages", - "description": "Maximum number of messages permitted in a batch.", - "type": "integer", - "default": 1000 - }, - "batching_max_publish_delay": { - "title": "Batching max publish delay", - "description": " Time period in milliseconds within which the messages sent will be batched.", - "type": "integer", - "default": 1 - }, - "block_if_queue_full": { - "title": "Block if queue is full", - "description": "If the send operation should block when the outgoing message queue is full.", - "type": "boolean", - "default": false - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "e06ad785-ad6f-4647-b2e8-3027a5c59454", - "name": "RabbitMQ", - "dockerRepository": "airbyte/destination-rabbitmq", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rabbitmq", - "icon": "pulsar.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rabbitmq", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Rabbitmq", - "type": "object", - "required": ["host", "routing_key"], - "additionalProperties": false, - "properties": { - "ssl": { - "type": "boolean", - "description": "SSL enabled.", - "default": true - }, - "host": { - "type": "string", - "description": "The RabbitMQ host name." - }, - "port": { - "type": "integer", - "description": "The RabbitMQ port." - }, - "virtual_host": { - "type": "string", - "description": "The RabbitMQ virtual host name." - }, - "username": { - "type": "string", - "description": "The username to connect." - }, - "password": { - "type": "string", - "description": "The password to connect." - }, - "exchange": { - "type": "string", - "description": "The exchange name." - }, - "routing_key": { - "type": "string", - "description": "The routing key." - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "d4d3fef9-e319-45c2-881a-bd02ce44cc9f", - "name": "Redis", - "dockerRepository": "airbyte/destination-redis", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redis", - "icon": "redis.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redis", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Redis Destination Spec", - "type": "object", - "required": ["host", "port", "username", "password", "cache_type"], - "additionalProperties": false, - "properties": { - "host": { - "title": "Host", - "description": "Redis host to connect to.", - "type": "string", - "examples": ["localhost,127.0.0.1"], - "order": 1 - }, - "port": { - "title": "Port", - "description": "Port of Redis.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 6379, - "order": 2 - }, - "username": { - "title": "Username", - "description": "Username associated with Redis.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "Password associated with Redis.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "cache_type": { - "title": "Cache type", - "type": "string", - "default": "hash", - "description": "Redis cache type to store data in.", - "enum": ["hash"], - "order": 5 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc", - "name": "Redshift", - "dockerRepository": "airbyte/destination-redshift", - "dockerImageTag": "0.3.47", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", - "icon": "redshift.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Redshift Destination Spec", - "type": "object", - "required": [ - "host", - "port", - "database", - "username", - "password", - "schema" - ], - "additionalProperties": true, - "properties": { - "host": { - "description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com)", - "type": "string", - "title": "Host" - }, - "port": { - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5439, - "examples": ["5439"], - "title": "Port" - }, - "username": { - "description": "Username to use to access the database.", - "type": "string", - "title": "Username" - }, - "password": { - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "title": "Password" - }, - "database": { - "description": "Name of the database.", - "type": "string", - "title": "Database" - }, - "schema": { - "description": "The default schema tables are written to if the source does not specify a namespace. Unless specifically configured, the usual value for this field is \"public\".", - "type": "string", - "examples": ["public"], - "default": "public", - "title": "Default Schema" - }, - "uploading_method": { - "title": "Uploading Method", - "type": "object", - "description": "The method how the data will be uploaded to the database.", - "oneOf": [ - { - "title": "Standard", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "S3 Staging", - "required": [ - "method", - "s3_bucket_name", - "s3_bucket_region", - "access_key_id", - "secret_access_key" - ], - "properties": { - "method": { - "type": "string", - "const": "S3 Staging" - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "The name of the staging S3 bucket to use if utilising a COPY strategy. COPY is recommended for production workloads for better speed and scalability. See AWS docs for more details.", - "examples": ["airbyte.staging"] - }, - "s3_bucket_path": { - "title": "S3 Bucket Path (Optional)", - "type": "string", - "description": "The directory under the S3 bucket where data will be written. If not provided, then defaults to the root directory. See path's name recommendations for more details.", - "examples": ["data_sync/test"] - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "The region of the S3 staging bucket to use if utilising a COPY strategy. See AWS docs for details.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1" - ] - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 8 - }, - "access_key_id": { - "type": "string", - "description": "This ID grants access to the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket. See AWS docs on how to generate an access key ID and secret access key.", - "title": "S3 Key Id", - "airbyte_secret": true - }, - "secret_access_key": { - "type": "string", - "description": "The corresponding secret to the above access key id. See AWS docs on how to generate an access key ID and secret access key.", - "title": "S3 Access Key", - "airbyte_secret": true - }, - "purge_staging_data": { - "title": "Purge Staging Files and Tables (Optional)", - "type": "boolean", - "description": "Whether to delete the staging files from S3 after completing the sync. See docs for details.", - "default": true - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "How to encrypt the staging data", - "default": { - "encryption_type": "none" - }, - "oneOf": [ - { - "title": "No encryption", - "description": "Staging data will be stored in plaintext.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "none", - "enum": ["none"], - "default": "none" - } - } - }, - { - "title": "AES-CBC envelope encryption", - "description": "Staging data will be encrypted using AES-CBC envelope encryption.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "aes_cbc_envelope", - "enum": ["aes_cbc_envelope"], - "default": "aes_cbc_envelope" - }, - "key_encrypting_key": { - "type": "string", - "title": "Key", - "description": "The key, base64-encoded. Must be either 128, 192, or 256 bits. Leave blank to have Airbyte generate an ephemeral key for each sync.", - "airbyte_secret": true - } - } - } - ] - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "beta", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "2c9d93a7-9a17-4789-9de9-f46f0097eb70", - "name": "Rockset", - "dockerRepository": "airbyte/destination-rockset", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rockset", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rockset", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Rockset Destination Spec", - "type": "object", - "required": ["api_key", "workspace"], - "additionalProperties": false, - "properties": { - "api_key": { - "title": "Api Key", - "description": "Rockset api key", - "type": "string", - "order": 0, - "airbyte_secret": true - }, - "workspace": { - "title": "Workspace", - "description": "The Rockset workspace in which collections will be created + written to.", - "type": "string", - "examples": ["commons", "my_workspace"], - "default": "commons", - "airbyte_secret": false, - "order": 1 - }, - "api_server": { - "title": "Api Server", - "description": "Rockset api URL", - "type": "string", - "airbyte_secret": false, - "default": "https://api.rs2.usw2.rockset.com", - "pattern": "^https:\\/\\/.*.rockset.com$", - "order": 2 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append", "overwrite"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "4816b78f-1489-44c1-9060-4b19d5fa9362", - "name": "S3", - "dockerRepository": "airbyte/destination-s3", - "dockerImageTag": "0.3.14", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3", - "icon": "s3.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3", - "protocol_version": "0.2.2", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "S3 Destination Spec", - "type": "object", - "required": [ - "s3_bucket_name", - "s3_bucket_path", - "s3_bucket_region", - "format" - ], - "properties": { - "access_key_id": { - "type": "string", - "description": "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", - "title": "S3 Key ID *", - "airbyte_secret": true, - "examples": ["A012345678910EXAMPLE"], - "order": 0 - }, - "secret_access_key": { - "type": "string", - "description": "The corresponding secret to the access key ID. Read more here", - "title": "S3 Access Key *", - "airbyte_secret": true, - "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"], - "order": 1 - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "The name of the S3 bucket. Read more here.", - "examples": ["airbyte_sync"], - "order": 2 - }, - "s3_bucket_path": { - "title": "S3 Bucket Path", - "description": "Directory under the S3 bucket where data will be written. Read more here", - "type": "string", - "examples": ["data_sync/test"], - "order": 3 - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "The region of the S3 bucket. See here for all region codes.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 4 - }, - "format": { - "title": "Output Format *", - "type": "object", - "description": "Format of the data output. See here for more details", - "oneOf": [ - { - "title": "Avro: Apache Avro", - "required": ["format_type", "compression_codec"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["Avro"], - "default": "Avro", - "order": 0 - }, - "compression_codec": { - "title": "Compression Codec *", - "description": "The compression algorithm used to compress data. Default to no compression.", - "type": "object", - "oneOf": [ - { - "title": "No Compression", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["no compression"], - "default": "no compression" - } - } - }, - { - "title": "Deflate", - "required": ["codec", "compression_level"], - "properties": { - "codec": { - "type": "string", - "enum": ["Deflate"], - "default": "Deflate" - }, - "compression_level": { - "title": "Deflate Level", - "description": "0: no compression & fastest, 9: best compression & slowest.", - "type": "integer", - "default": 0, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "bzip2", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["bzip2"], - "default": "bzip2" - } - } - }, - { - "title": "xz", - "required": ["codec", "compression_level"], - "properties": { - "codec": { - "type": "string", - "enum": ["xz"], - "default": "xz" - }, - "compression_level": { - "title": "Compression Level", - "description": "See here for details.", - "type": "integer", - "default": 6, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "zstandard", - "required": ["codec", "compression_level"], - "properties": { - "codec": { - "type": "string", - "enum": ["zstandard"], - "default": "zstandard" - }, - "compression_level": { - "title": "Compression Level", - "description": "Negative levels are 'fast' modes akin to lz4 or snappy, levels above 9 are generally for archival purposes, and levels above 18 use a lot of memory.", - "type": "integer", - "default": 3, - "minimum": -5, - "maximum": 22 - }, - "include_checksum": { - "title": "Include Checksum", - "description": "If true, include a checksum with each data block.", - "type": "boolean", - "default": false - } - } - }, - { - "title": "snappy", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["snappy"], - "default": "snappy" - } - } - } - ], - "order": 1 - } - } - }, - { - "title": "CSV: Comma-Separated Values", - "required": ["format_type", "flattening"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["CSV"], - "default": "CSV" - }, - "flattening": { - "type": "string", - "title": "Normalization (Flattening)", - "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "JSON Lines: Newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["JSONL"], - "default": "JSONL" - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "Parquet: Columnar Storage", - "required": ["format_type"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["Parquet"], - "default": "Parquet" - }, - "compression_codec": { - "title": "Compression Codec (Optional)", - "description": "The compression algorithm used to compress data pages.", - "type": "string", - "enum": [ - "UNCOMPRESSED", - "SNAPPY", - "GZIP", - "LZO", - "BROTLI", - "LZ4", - "ZSTD" - ], - "default": "UNCOMPRESSED" - }, - "block_size_mb": { - "title": "Block Size (Row Group Size) (MB) (Optional)", - "description": "This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. Default: 128 MB.", - "type": "integer", - "default": 128, - "examples": [128] - }, - "max_padding_size_mb": { - "title": "Max Padding Size (MB) (Optional)", - "description": "Maximum size allowed as padding to align row groups. This is also the minimum size of a row group. Default: 8 MB.", - "type": "integer", - "default": 8, - "examples": [8] - }, - "page_size_kb": { - "title": "Page Size (KB) (Optional)", - "description": "The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_page_size_kb": { - "title": "Dictionary Page Size (KB) (Optional)", - "description": "There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_encoding": { - "title": "Dictionary Encoding (Optional)", - "description": "Default: true.", - "type": "boolean", - "default": true - } - } - } - ], - "order": 5 - }, - "s3_endpoint": { - "title": "Endpoint (Optional)", - "type": "string", - "default": "", - "description": "Your S3 endpoint url. Read more here", - "examples": ["http://localhost:9000"], - "order": 6 - }, - "s3_path_format": { - "title": "S3 Path Format (Optional)", - "description": "Format string on how data will be organized inside the S3 bucket directory. Read more here", - "type": "string", - "examples": [ - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_" - ], - "order": 7 - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 8 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "e9810f61-4bab-46d2-bb22-edfc902e0644", - "name": "SFTP-JSON", - "dockerRepository": "airbyte/destination-sftp-json", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/sftp-json", - "icon": "sftp.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/sftp-json", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination SFTP JSON", - "type": "object", - "required": ["host", "username", "password", "destination_path"], - "additionalProperties": false, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the SFTP server.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the SFTP server.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": [22], - "order": 1 - }, - "username": { - "title": "User", - "description": "Username to use to access the SFTP server.", - "type": "string", - "order": 2 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 3 - }, - "destination_path": { - "title": "Destination path", - "type": "string", - "description": "Path to the directory where json files will be written.", - "examples": ["/json_data"], - "order": 4 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "3dc6f384-cd6b-4be3-ad16-a41450899bf0", - "name": "Scylla", - "dockerRepository": "airbyte/destination-scylla", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/scylla", - "icon": "scylla.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/scylla", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Scylla Destination Spec", - "type": "object", - "required": ["keyspace", "username", "password", "address", "port"], - "additionalProperties": true, - "properties": { - "keyspace": { - "title": "Keyspace", - "description": "Default Scylla keyspace to create data in.", - "type": "string", - "order": 0 - }, - "username": { - "title": "Username", - "description": "Username to use to access Scylla.", - "type": "string", - "order": 1 - }, - "password": { - "title": "Password", - "description": "Password associated with Scylla.", - "type": "string", - "airbyte_secret": true, - "order": 2 - }, - "address": { - "title": "Address", - "description": "Address to connect to.", - "type": "string", - "order": 3 - }, - "port": { - "title": "Port", - "description": "Port of Scylla.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9042, - "order": 4 - }, - "replication": { - "title": "Replication factor", - "type": "integer", - "description": "Indicates to how many nodes the data should be replicated to.", - "default": 1, - "order": 5 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", - "name": "Snowflake", - "dockerRepository": "airbyte/destination-snowflake", - "dockerImageTag": "0.4.34", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", - "icon": "snowflake.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Snowflake Destination Spec", - "type": "object", - "required": [ - "host", - "role", - "warehouse", - "database", - "schema", - "username" - ], - "additionalProperties": true, - "properties": { - "host": { - "description": "Enter your Snowflake account's locator (in the format ...snowflakecomputing.com)", - "examples": [ - "accountname.us-east-2.aws.snowflakecomputing.com", - "accountname.snowflakecomputing.com" - ], - "type": "string", - "title": "Host", - "order": 0 - }, - "role": { - "description": "Enter the role that you want to use to access Snowflake", - "examples": ["AIRBYTE_ROLE"], - "type": "string", - "title": "Role", - "order": 1 - }, - "warehouse": { - "description": "Enter the name of the warehouse that you want to sync data into", - "examples": ["AIRBYTE_WAREHOUSE"], - "type": "string", - "title": "Warehouse", - "order": 2 - }, - "database": { - "description": "Enter the name of the database you want to sync data into", - "examples": ["AIRBYTE_DATABASE"], - "type": "string", - "title": "Database", - "order": 3 - }, - "schema": { - "description": "Enter the name of the default schema", - "examples": ["AIRBYTE_SCHEMA"], - "type": "string", - "title": "Default Schema", - "order": 4 - }, - "username": { - "description": "Enter the name of the user you want to use to access the database", - "examples": ["AIRBYTE_USER"], - "type": "string", - "title": "Username", - "order": 5 - }, - "credentials": { - "title": "Authorization Method", - "description": "", - "type": "object", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "order": 0, - "required": ["access_token", "refresh_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "OAuth2.0", - "enum": ["OAuth2.0"], - "default": "OAuth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "Enter your application's Client ID", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "Enter your application's Client secret", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Enter you application's Access Token", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Enter your application's Refresh Token", - "airbyte_secret": true - } - } - }, - { - "title": "Key Pair Authentication", - "type": "object", - "order": 1, - "required": ["private_key"], - "properties": { - "auth_type": { - "type": "string", - "const": "Key Pair Authentication", - "enum": ["Key Pair Authentication"], - "default": "Key Pair Authentication", - "order": 0 - }, - "private_key": { - "type": "string", - "title": "Private Key", - "description": "RSA Private key to use for Snowflake connection. See the docs for more information on how to obtain this key.", - "multiline": true, - "airbyte_secret": true - }, - "private_key_password": { - "type": "string", - "title": "Passphrase (Optional)", - "description": "Passphrase for private key", - "airbyte_secret": true - } - } - }, - { - "title": "Username and Password", - "type": "object", - "required": ["password"], - "order": 2, - "properties": { - "password": { - "description": "Enter the password associated with the username.", - "type": "string", - "airbyte_secret": true, - "title": "Password", - "order": 1 - } - } - } - ], - "order": 6 - }, - "jdbc_url_params": { - "description": "Enter the additional properties to pass to the JDBC URL string when connecting to the database (formatted as key=value pairs separated by the symbol &). Example: key1=value1&key2=value2&key3=value3", - "title": "JDBC URL Params", - "type": "string", - "order": 7 - }, - "loading_method": { - "type": "object", - "title": "Data Staging Method", - "description": "Select a data staging method", - "order": 8, - "oneOf": [ - { - "title": "Select another option", - "description": "Select another option", - "required": ["method"], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Standard"], - "default": "Standard" - } - } - }, - { - "title": "[Recommended] Internal Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": ["method"], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Internal Staging"], - "default": "Internal Staging" - } - } - }, - { - "title": "AWS S3 Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "s3_bucket_name", - "access_key_id", - "secret_access_key" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["S3 Staging"], - "default": "S3 Staging", - "order": 0 - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "Enter your S3 bucket name", - "examples": ["airbyte.staging"], - "order": 1 - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "Enter the region where your S3 bucket resides", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "eu-south-1", - "eu-north-1", - "sa-east-1", - "me-south-1" - ], - "order": 2 - }, - "access_key_id": { - "type": "string", - "description": "Enter your AWS access key ID. Airbyte requires Read and Write permissions on your S3 bucket ", - "title": "AWS access key ID", - "airbyte_secret": true, - "order": 3 - }, - "secret_access_key": { - "type": "string", - "description": "Enter your AWS secret access key", - "title": "AWS secret access key", - "airbyte_secret": true, - "order": 4 - }, - "purge_staging_data": { - "title": "Purge Staging Files and Tables", - "type": "boolean", - "description": "Toggle to delete staging files from the S3 bucket after a successful sync", - "default": true, - "order": 5 - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "Choose a data encryption method for the staging data", - "default": { - "encryption_type": "none" - }, - "order": 6, - "oneOf": [ - { - "title": "No encryption", - "description": "Staging data will be stored in plaintext.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "none", - "enum": ["none"], - "default": "none" - } - } - }, - { - "title": "AES-CBC envelope encryption", - "description": "Staging data will be encrypted using AES-CBC envelope encryption.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "aes_cbc_envelope", - "enum": ["aes_cbc_envelope"], - "default": "aes_cbc_envelope" - }, - "key_encrypting_key": { - "type": "string", - "title": "Key", - "description": "The key, base64-encoded. Must be either 128, 192, or 256 bits. Leave blank to have Airbyte generate an ephemeral key for each sync.", - "airbyte_secret": true - } - } - } - ] - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 7 - } - } - }, - { - "title": "Google Cloud Storage Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "project_id", - "bucket_name", - "credentials_json" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["GCS Staging"], - "default": "GCS Staging", - "order": 0 - }, - "project_id": { - "title": "Google Cloud project ID", - "type": "string", - "description": "Enter the Google Cloud project ID", - "examples": ["my-project"], - "order": 1 - }, - "bucket_name": { - "title": "Cloud Storage bucket name", - "type": "string", - "description": "Enter the Cloud Storage bucket name", - "examples": ["airbyte-staging"], - "order": 2 - }, - "credentials_json": { - "title": "Google Application Credentials", - "type": "string", - "description": "Enter your Google Cloud service account key in the JSON format with read/write access to your Cloud Storage staging bucket", - "airbyte_secret": true, - "multiline": true, - "order": 3 - } - } - }, - { - "title": "Azure Blob Storage Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "azure_blob_storage_account_name", - "azure_blob_storage_container_name", - "azure_blob_storage_sas_token" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Azure Blob Staging"], - "default": "Azure Blob Staging", - "order": 0 - }, - "azure_blob_storage_endpoint_domain_name": { - "title": "Azure Blob Storage Endpoint", - "type": "string", - "default": "blob.core.windows.net", - "description": "Enter the Azure Blob Storage endpoint domain name", - "examples": ["blob.core.windows.net"], - "order": 1 - }, - "azure_blob_storage_account_name": { - "title": "Azure Blob Storage account name", - "type": "string", - "description": "Enter your Azure Blob Storage account name", - "examples": ["airbyte5storage"], - "order": 2 - }, - "azure_blob_storage_container_name": { - "title": "Azure Blob Storage Container Name", - "type": "string", - "description": "Enter your Azure Blob Storage container name", - "examples": ["airbytetestcontainername"], - "order": 3 - }, - "azure_blob_storage_sas_token": { - "title": "SAS Token", - "type": "string", - "airbyte_secret": true, - "description": "Enter the Shared access signature (SAS) token to grant Snowflake limited access to objects in your Azure Blob Storage account", - "examples": [ - "?sv=2016-05-31&ss=b&srt=sco&sp=rwdl&se=2018-06-27T10:05:50Z&st=2017-06-27T02:05:50Z&spr=https,http&sig=bgqQwoXwxzuD2GJfagRg7VOS8hzNr3QLT7rhS8OFRLQ%3D" - ], - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "OAuth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "host": { - "type": "string", - "path_in_connector_config": ["host"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - } - ], - "sources": [ - { - "sourceDefinitionId": "6ff047c0-f5d5-4ce5-8c81-204a830fa7e1", - "name": "AWS CloudTrail", - "dockerRepository": "airbyte/source-aws-cloudtrail", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/aws-cloudtrail", - "icon": "awscloudtrail.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/aws-cloudtrail", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Aws CloudTrail Spec", - "type": "object", - "required": [ - "aws_key_id", - "aws_secret_key", - "aws_region_name", - "start_date" - ], - "additionalProperties": true, - "properties": { - "aws_key_id": { - "type": "string", - "title": "Key ID", - "description": "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, - "aws_secret_key": { - "type": "string", - "title": "Secret Key", - "description": "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, - "aws_region_name": { - "type": "string", - "title": "Region Name", - "description": "The default AWS Region to use, for example, us-west-1 or us-west-2. When specifying a Region inline during client initialization, this property is named region_name." - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date you would like to replicate data. Data in AWS CloudTrail is available for last 90 days only. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "default": "1970-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212", - "name": "Airtable", - "dockerRepository": "airbyte/source-airtable", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/airtable", - "icon": "airtable.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/airtable", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Airtable Source Spec", - "type": "object", - "required": ["api_key", "base_id", "tables"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "The API Key for the Airtable account. See the Support Guide for more information on how to obtain this key.", - "title": "API Key", - "airbyte_secret": true, - "examples": ["key1234567890"] - }, - "base_id": { - "type": "string", - "description": "The Base ID to integrate the data from. You can find the Base ID following the link Airtable API, log in to your account, select the base you need and find Base ID in the docs.", - "title": "Base ID", - "examples": ["app1234567890"] - }, - "tables": { - "type": "array", - "items": { - "type": "string" - }, - "description": "The list of Tables to integrate.", - "title": "Tables", - "examples": ["table 1", "table 2"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246", - "name": "Amazon Ads", - "dockerRepository": "airbyte/source-amazon-ads", - "dockerImageTag": "0.1.18", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-ads", - "icon": "amazonads.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/amazon-ads", - "connectionSpecification": { - "title": "Amazon Ads Spec", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "client_id": { - "title": "Client ID", - "description": "The client ID of your Amazon Ads developer application. See the docs for more information.", - "order": 1, - "type": "string" - }, - "client_secret": { - "title": "Client Secret", - "description": "The client secret of your Amazon Ads developer application. See the docs for more information.", - "airbyte_secret": true, - "order": 2, - "type": "string" - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Amazon Ads refresh token. See the docs for more information on how to obtain this token.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "region": { - "title": "Region *", - "description": "Region to pull data from (EU/NA/FE). See docs for more details.", - "enum": ["NA", "EU", "FE"], - "type": "string", - "default": "NA", - "order": 4 - }, - "report_wait_timeout": { - "title": "Report Wait Timeout *", - "description": "Timeout duration in minutes for Reports. Default is 30 minutes.", - "default": 30, - "examples": [30, 120], - "order": 5, - "type": "integer" - }, - "report_generation_max_retries": { - "title": "Report Generation Maximum Retries *", - "description": "Maximum retries Airbyte will attempt for fetching report data. Default is 5.", - "default": 5, - "examples": [5, 10, 15], - "order": 6, - "type": "integer" - }, - "start_date": { - "title": "Start Date (Optional)", - "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format", - "examples": ["2022-10-10", "2022-10-22"], - "order": 7, - "type": "string" - }, - "profiles": { - "title": "Profile IDs (Optional)", - "description": "Profile IDs you want to fetch data for. See docs for more details.", - "order": 8, - "type": "array", - "items": { - "type": "integer" - } - } - }, - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "983fd355-6bf3-4709-91b5-37afa391eeb6", - "name": "Amazon SQS", - "dockerRepository": "airbyte/source-amazon-sqs", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-sqs", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-sqs", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Amazon SQS Source Spec", - "type": "object", - "required": ["queue_url", "region", "delete_messages"], - "additionalProperties": false, - "properties": { - "queue_url": { - "title": "Queue URL", - "description": "URL of the SQS Queue", - "type": "string", - "examples": [ - "https://sqs.eu-west-1.amazonaws.com/1234567890/my-example-queue" - ], - "order": 0 - }, - "region": { - "title": "AWS Region", - "description": "AWS Region of the SQS Queue", - "type": "string", - "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 1 - }, - "delete_messages": { - "title": "Delete Messages After Read", - "description": "If Enabled, messages will be deleted from the SQS Queue after being read. If Disabled, messages are left in the queue and can be read more than once. WARNING: Enabling this option can result in data loss in cases of failure, use with caution, see documentation for more detail. ", - "type": "boolean", - "default": false, - "order": 2 - }, - "max_batch_size": { - "title": "Max Batch Size", - "description": "Max amount of messages to get in one batch (10 max)", - "type": "integer", - "examples": ["5"], - "order": 3 - }, - "max_wait_time": { - "title": "Max Wait Time", - "description": "Max amount of time in seconds to wait for messages in a single poll (20 max)", - "type": "integer", - "examples": ["5"], - "order": 4 - }, - "attributes_to_return": { - "title": "Message Attributes To Return", - "description": "Comma separated list of Mesage Attribute names to return", - "type": "string", - "examples": ["attr1,attr2"], - "order": 5 - }, - "visibility_timeout": { - "title": "Message Visibility Timeout", - "description": "Modify the Visibility Timeout of the individual message from the Queue's default (seconds).", - "type": "integer", - "examples": ["15"], - "order": 6 - }, - "access_key": { - "title": "AWS IAM Access Key ID", - "description": "The Access Key ID of the AWS IAM Role to use for pulling messages", - "type": "string", - "examples": ["xxxxxHRNxxx3TBxxxxxx"], - "airbyte_secret": true, - "order": 7 - }, - "secret_key": { - "title": "AWS IAM Secret Key", - "description": "The Secret Key of the AWS IAM Role to use for pulling messages", - "type": "string", - "examples": ["hu+qE5exxxxT6o/ZrKsxxxxxxBhxxXLexxxxxVKz"], - "airbyte_secret": true, - "order": 8 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e55879a8-0ef8-4557-abcf-ab34c53ec460", - "name": "Amazon Seller Partner", - "dockerRepository": "airbyte/source-amazon-seller-partner", - "dockerImageTag": "0.2.25", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner", - "icon": "amazonsellerpartner.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner", - "connectionSpecification": { - "title": "Amazon Seller Partner Spec", - "type": "object", - "properties": { - "app_id": { - "title": "App Id *", - "description": "Your Amazon App ID", - "airbyte_secret": true, - "order": 0, - "type": "string" - }, - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 1, - "type": "string" - }, - "lwa_app_id": { - "title": "LWA Client Id", - "description": "Your Login with Amazon Client ID.", - "order": 2, - "type": "string" - }, - "lwa_client_secret": { - "title": "LWA Client Secret", - "description": "Your Login with Amazon Client Secret.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "refresh_token": { - "title": "Refresh Token", - "description": "The Refresh Token obtained via OAuth flow authorization.", - "airbyte_secret": true, - "order": 4, - "type": "string" - }, - "aws_access_key": { - "title": "AWS Access Key", - "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 5, - "type": "string" - }, - "aws_secret_key": { - "title": "AWS Secret Access Key", - "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 6, - "type": "string" - }, - "role_arn": { - "title": "Role ARN", - "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", - "airbyte_secret": true, - "order": 7, - "type": "string" - }, - "replication_start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string" - }, - "replication_end_date": { - "title": "End Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string" - }, - "period_in_days": { - "title": "Period In Days", - "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", - "default": 30, - "examples": ["30", "365"], - "type": "integer" - }, - "report_options": { - "title": "Report Options", - "description": "Additional information passed to reports. This varies by report type. Must be a valid json string.", - "examples": [ - "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "type": "string" - }, - "max_wait_seconds": { - "title": "Max wait time for reports (in seconds)", - "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", - "default": 500, - "examples": ["500", "1980"], - "type": "integer" - }, - "aws_environment": { - "title": "AWSEnvironment", - "description": "An enumeration.", - "enum": ["PRODUCTION", "SANDBOX"], - "type": "string" - }, - "region": { - "title": "AWSRegion", - "description": "An enumeration.", - "enum": [ - "AE", - "AU", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], - "type": "string" - } - }, - "required": [ - "lwa_app_id", - "lwa_client_secret", - "refresh_token", - "aws_access_key", - "aws_secret_key", - "role_arn", - "replication_start_date", - "aws_environment", - "region" - ], - "additionalProperties": true, - "definitions": { - "AWSEnvironment": { - "title": "AWSEnvironment", - "description": "An enumeration.", - "enum": ["PRODUCTION", "SANDBOX"], - "type": "string" - }, - "AWSRegion": { - "title": "AWSRegion", - "description": "An enumeration.", - "enum": [ - "AE", - "AU", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], - "type": "string" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "app_id": { - "type": "string", - "path_in_connector_config": ["app_id"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string" - }, - "lwa_client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string", - "path_in_connector_config": ["lwa_app_id"] - }, - "lwa_client_secret": { - "type": "string", - "path_in_connector_config": ["lwa_client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "fa9f58c6-2d03-4237-aaa4-07d75e0c1396", - "name": "Amplitude", - "dockerRepository": "airbyte/source-amplitude", - "dockerImageTag": "0.1.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amplitude", - "icon": "amplitude.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amplitude", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Amplitude Spec", - "type": "object", - "required": ["api_key", "secret_key", "start_date"], - "additionalProperties": true, - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "description": "Amplitude API Key. See the setup guide for more information on how to obtain this key.", - "airbyte_secret": true - }, - "secret_key": { - "type": "string", - "title": "Secret Key", - "description": "Amplitude Secret Key. See the setup guide for more information on how to obtain this key.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Replication Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-01-25T00:00:00Z"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "47f17145-fe20-4ef5-a548-e29b048adf84", - "name": "Apify Dataset", - "dockerRepository": "airbyte/source-apify-dataset", - "dockerImageTag": "0.1.11", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/apify-dataset", - "icon": "apify.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/apify-dataset", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Apify Dataset Spec", - "type": "object", - "required": ["datasetId"], - "additionalProperties": false, - "properties": { - "datasetId": { - "type": "string", - "title": "Dataset ID", - "description": "ID of the dataset you would like to load to Airbyte." - }, - "clean": { - "type": "boolean", - "title": "Clean", - "description": "If set to true, only clean items will be downloaded from the dataset. See description of what clean means in Apify API docs. If not sure, set clean to false." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "798ae795-5189-42b6-b64e-3cb91db93338", - "name": "Azure Table Storage", - "dockerRepository": "airbyte/source-azure-table", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/azure-table", - "icon": "azureblobstorage.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Azure Data Table Spec", - "type": "object", - "required": ["storage_account_name", "storage_access_key"], - "additionalProperties": false, - "properties": { - "storage_account_name": { - "title": "Account Name", - "type": "string", - "description": "The name of your storage account.", - "order": 0, - "airbyte_secret": false - }, - "storage_access_key": { - "title": "Access Key", - "type": "string", - "description": "Azure Table Storage Access Key. See the docs for more information on how to obtain this key.", - "order": 1, - "airbyte_secret": true - }, - "storage_endpoint_suffix": { - "title": "Endpoint Suffix", - "type": "string", - "description": "Azure Table Storage service account URL suffix. See the docs for more information on how to obtain endpoint suffix", - "order": 2, - "default": "core.windows.net", - "examples": ["core.windows.net", "core.chinacloudapi.cn"], - "airbyte_secret": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "59c5501b-9f95-411e-9269-7143c939adbd", - "name": "BigCommerce", - "dockerRepository": "airbyte/source-bigcommerce", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigcommerce", - "icon": "bigcommerce.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigcommerce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigCommerce Source CDK Specifications", - "type": "object", - "required": ["start_date", "store_hash", "access_token"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date you would like to replicate data. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "store_hash": { - "type": "string", - "title": "Store Hash", - "description": "The hash code of the store. For https://api.bigcommerce.com/stores/HASH_CODE/v3/, The store's hash code is 'HASH_CODE'." - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c", - "name": "BigQuery", - "dockerRepository": "airbyte/source-bigquery", - "dockerImageTag": "0.2.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigquery", - "icon": "bigquery.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigquery", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Source Spec", - "type": "object", - "required": ["project_id", "credentials_json"], - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset.", - "title": "Project ID" - }, - "dataset_id": { - "type": "string", - "description": "The dataset ID to search for tables and views. If you are only loading data from one dataset, setting this option could result in much faster schema discovery.", - "title": "Default Dataset ID" - }, - "credentials_json": { - "type": "string", - "description": "The contents of your Service Account Key JSON file. See the docs for more information on how to obtain this key.", - "title": "Credentials JSON", - "airbyte_secret": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [], - "supported_sync_modes": ["overwrite", "append", "append_dedup"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "47f25999-dd5e-4636-8c39-e7cea2453331", - "name": "Bing Ads", - "dockerRepository": "airbyte/source-bing-ads", - "dockerImageTag": "0.1.10", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bing-ads", - "icon": "bingads.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bing-ads", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Bing Ads Spec", - "type": "object", - "required": [ - "developer_token", - "client_id", - "refresh_token", - "reports_start_date" - ], - "additionalProperties": true, - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0" - }, - "tenant_id": { - "type": "string", - "title": "Tenant ID", - "description": "The Tenant ID of your Microsoft Advertising developer application. Set this to \"common\" unless you know you need a different value.", - "airbyte_secret": true, - "default": "common", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 1 - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Microsoft Advertising developer application.", - "default": "", - "airbyte_secret": true, - "order": 2 - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token.", - "airbyte_secret": true, - "order": 3 - }, - "developer_token": { - "type": "string", - "title": "Developer Token", - "description": "Developer token associated with user. See more info in the docs.", - "airbyte_secret": true, - "order": 4 - }, - "reports_start_date": { - "type": "string", - "title": "Reports replication start date", - "format": "date", - "default": "2020-01-01", - "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 5 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "tenant_id": { - "type": "string", - "path_in_connector_config": ["tenant_id"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "63cea06f-1c75-458d-88fe-ad48c7cb27fd", - "name": "Braintree", - "dockerRepository": "airbyte/source-braintree", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/braintree", - "icon": "braintree.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/braintree", - "connectionSpecification": { - "title": "Braintree Spec", - "type": "object", - "properties": { - "merchant_id": { - "title": "Merchant ID", - "description": "The unique identifier for your entire gateway account. See the docs for more information on how to obtain this ID.", - "name": "Merchant ID", - "type": "string" - }, - "public_key": { - "title": "Public Key", - "description": "Braintree Public Key. See the docs for more information on how to obtain this key.", - "name": "Public Key", - "type": "string" - }, - "private_key": { - "title": "Private Key", - "description": "Braintree Private Key. See the docs for more information on how to obtain this key.", - "name": "Private Key", - "airbyte_secret": true, - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "name": "Start Date", - "examples": ["2020", "2020-12-30", "2020-11-22 20:20:05"], - "type": "string", - "format": "date-time" - }, - "environment": { - "title": "Environment", - "description": "Environment specifies where the data will come from.", - "name": "Environment", - "examples": ["sandbox", "production", "qa", "development"], - "enum": ["Development", "Sandbox", "Qa", "Production"], - "type": "string" - } - }, - "required": [ - "merchant_id", - "public_key", - "private_key", - "environment" - ] - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "686473f1-76d9-4994-9cc7-9b13da46147c", - "name": "Chargebee", - "dockerRepository": "airbyte/source-chargebee", - "dockerImageTag": "0.1.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/chargebee", - "icon": "chargebee.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://apidocs.chargebee.com/docs/api", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Chargebee Spec", - "type": "object", - "required": ["site", "site_api_key", "start_date", "product_catalog"], - "additionalProperties": true, - "properties": { - "site": { - "type": "string", - "title": "Site", - "description": "The site prefix for your Chargebee instance.", - "examples": ["airbyte-test"] - }, - "site_api_key": { - "type": "string", - "title": "API Key", - "description": "Chargebee API Key. See the docs for more information on how to obtain this key.", - "examples": ["test_3yzfanAXF66USdWC9wQcM555DQJkSYoppu"], - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-01-25T00:00:00Z"] - }, - "product_catalog": { - "title": "Product Catalog", - "type": "string", - "description": "Product Catalog version of your Chargebee site. Instructions on how to find your version you may find here under `API Version` section.", - "enum": ["1.0", "2.0"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "b6604cbd-1b12-4c08-8767-e140d0fb0877", - "name": "Chartmogul", - "dockerRepository": "airbyte/source-chartmogul", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/chartmogul", - "icon": "chartmogul.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/chartmogul", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Chartmogul Spec", - "type": "object", - "required": ["api_key", "start_date", "interval"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "Chartmogul API key", - "airbyte_secret": true, - "order": 0 - }, - "start_date": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. When feasible, any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"], - "order": 1 - }, - "interval": { - "type": "string", - "description": "Some APIs such as Metrics require intervals to cluster data.", - "enum": ["day", "week", "month", "quarter"], - "default": "month", - "order": 2 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "bad83517-5e54-4a3d-9b53-63e85fbd4d7c", - "name": "ClickHouse", - "dockerRepository": "airbyte/source-clickhouse-strict-encrypt", - "dockerImageTag": "0.1.8", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse", - "icon": "clickhouse.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ClickHouse Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "description": "The host endpoint of the Clickhouse cluster.", - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of the database.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 8123, - "examples": ["8123"] - }, - "database": { - "description": "The name of the database.", - "title": "Database", - "type": "string", - "examples": ["default"] - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string" - }, - "password": { - "description": "The password associated with this username.", - "title": "Password", - "type": "string", - "airbyte_secret": true - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "dfffecb7-9a13-43e9-acdc-b92af7997ca9", - "name": "Close.com", - "dockerRepository": "airbyte/source-close-com", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/close-com", - "icon": "close.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/close-com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Close.com Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "Close.com API key (usually starts with 'api_'; find yours here).", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "description": "The start date to sync data. Leave blank for full sync. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "default": "2021-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "cc88c43f-6f53-4e8a-8c4d-b284baaf9635", - "name": "Delighted", - "dockerRepository": "airbyte/source-delighted", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/delighted", - "icon": "delighted.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Delighted Spec", - "type": "object", - "required": ["since", "api_key"], - "additionalProperties": false, - "properties": { - "since": { - "title": "Since", - "type": "string", - "description": "The date from which you'd like to replicate the data", - "examples": ["2022-05-30 04:50:23"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$", - "order": 0 - }, - "api_key": { - "title": "Delighted API Key", - "type": "string", - "description": "A Delighted API key.", - "airbyte_secret": true, - "order": 1 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "0b5c867e-1b12-4d02-ab74-97b2184ff6d7", - "name": "Dixa", - "dockerRepository": "airbyte/source-dixa", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dixa", - "icon": "dixa.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dixa", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Dixa Spec", - "type": "object", - "required": ["api_token", "start_date"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "description": "Dixa API token", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "description": "The connector pulls records updated from this date onwards.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["YYYY-MM-DD"] - }, - "batch_size": { - "type": "integer", - "description": "Number of days to batch into one request. Max 31.", - "pattern": "^[0-9]{1,2}$", - "examples": [1, 31], - "default": 31 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "72d405a3-56d8-499f-a571-667c03406e43", - "name": "Dockerhub", - "dockerRepository": "airbyte/source-dockerhub", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dockerhub", - "icon": "dockerhub.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dockerhub", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Dockerhub Spec", - "type": "object", - "required": ["docker_username"], - "additionalProperties": false, - "properties": { - "docker_username": { - "type": "string", - "description": "Username of DockerHub person or organization (for https://hub.docker.com/v2/repositories/USERNAME/ API call)", - "pattern": "^[a-z0-9_\\-]+$", - "examples": ["airbyte"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "50bd8338-7c4e-46f1-8c7f-3ef95de19fdd", - "name": "End-to-End Testing (Mock API)", - "dockerRepository": "airbyte/source-e2e-test-cloud", - "dockerImageTag": "2.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/e2e-test", - "icon": "airbyte.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/e2e-test", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Cloud E2E Test Source Spec", - "type": "object", - "required": ["max_messages", "mock_catalog"], - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "const": "CONTINUOUS_FEED", - "default": "CONTINUOUS_FEED", - "order": 10 - }, - "max_messages": { - "title": "Max Records", - "description": "Number of records to emit per stream. Min 1. Max 100 billion.", - "type": "integer", - "default": 100, - "min": 1, - "max": 100000000000, - "order": 20 - }, - "seed": { - "title": "Random Seed", - "description": "When the seed is unspecified, the current time millis will be used as the seed. Range: [0, 1000000].", - "type": "integer", - "default": 0, - "examples": [42], - "min": 0, - "max": 1000000, - "order": 30 - }, - "message_interval_ms": { - "title": "Message Interval (ms)", - "description": "Interval between messages in ms. Min 0 ms. Max 60000 ms (1 minute).", - "type": "integer", - "min": 0, - "max": 60000, - "default": 0, - "order": 40 - }, - "mock_catalog": { - "title": "Mock Catalog", - "type": "object", - "order": 50, - "oneOf": [ - { - "title": "Single Schema", - "description": "A catalog with one or multiple streams that share the same schema.", - "required": ["type", "stream_name", "stream_schema"], - "properties": { - "type": { - "type": "string", - "const": "SINGLE_STREAM", - "default": "SINGLE_STREAM" - }, - "stream_name": { - "title": "Stream Name", - "description": "Name of the data stream.", - "type": "string", - "default": "data_stream" - }, - "stream_schema": { - "title": "Stream Schema", - "description": "A Json schema for the stream. The schema should be compatible with draft-07. See this doc for examples.", - "type": "string", - "default": "{ \"type\": \"object\", \"properties\": { \"column1\": { \"type\": \"string\" } } }" - }, - "stream_duplication": { - "title": "Duplicate the stream N times", - "description": "Duplicate the stream for easy load testing. Each stream name will have a number suffix. For example, if the stream name is \"ds\", the duplicated streams will be \"ds_0\", \"ds_1\", etc.", - "type": "integer", - "default": 1, - "min": 1, - "max": 10000 - } - } - }, - { - "title": "Multi Schema", - "description": "A catalog with multiple data streams, each with a different schema.", - "required": ["type", "stream_schemas"], - "properties": { - "type": { - "type": "string", - "const": "MULTI_STREAM", - "default": "MULTI_STREAM" - }, - "stream_schemas": { - "title": "Streams and Schemas", - "description": "A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. The schema should be compatible with draft-07. See this doc for examples.", - "type": "string", - "default": "{ \"stream1\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" } } }, \"stream2\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"boolean\" } } } }" - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e2b40e36-aa0e-4bed-b41b-bcea6fa348b1", - "name": "Exchange Rates Api", - "dockerRepository": "airbyte/source-exchange-rates", - "dockerImageTag": "0.2.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi", - "icon": "exchangeratesapi.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "exchangeratesapi.io Source Spec", - "type": "object", - "required": ["start_date", "access_key"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "description": "Start getting data from that date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["YYYY-MM-DD"] - }, - "access_key": { - "type": "string", - "description": "Your API Access Key. See here. The key is case sensitive.", - "airbyte_secret": true - }, - "base": { - "type": "string", - "description": "ISO reference currency. See here. Free plan doesn't support Source Currency Switching, default base currency is EUR", - "examples": ["EUR", "USD"] - }, - "ignore_weekends": { - "type": "boolean", - "description": "Ignore weekends? (Exchanges don't run on weekends)", - "default": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", - "name": "Facebook Marketing", - "dockerRepository": "airbyte/source-facebook-marketing", - "dockerImageTag": "0.2.60", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", - "icon": "facebook.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", - "connectionSpecification": { - "title": "Source Facebook Marketing", - "type": "object", - "properties": { - "account_id": { - "title": "Account ID", - "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API.", - "order": 0, - "examples": ["111111111111111"], - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "order": 1, - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the latest data.", - "order": 2, - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-26T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "access_token": { - "title": "Access Token", - "description": "The value of the access token generated. See the docs for more information", - "order": 3, - "airbyte_secret": true, - "type": "string" - }, - "include_deleted": { - "title": "Include Deleted", - "description": "Include data from deleted Campaigns, Ads, and AdSets", - "default": false, - "order": 4, - "type": "boolean" - }, - "fetch_thumbnail_images": { - "title": "Fetch Thumbnail Images", - "description": "In each Ad Creative, fetch the thumbnail_url and store the result in thumbnail_data_url", - "default": false, - "order": 5, - "type": "boolean" - }, - "custom_insights": { - "title": "Custom Insights", - "description": "A list which contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)", - "order": 6, - "type": "array", - "items": { - "title": "InsightConfig", - "description": "Config for custom insights", - "type": "object", - "properties": { - "name": { - "title": "Name", - "description": "The name value of insight", - "type": "string" - }, - "fields": { - "title": "Fields", - "description": "A list of chosen fields for fields parameter", - "default": [], - "type": "array", - "items": { - "title": "ValidEnums", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", - "enum": [ - "account_currency", - "account_id", - "account_name", - "action_values", - "actions", - "ad_bid_value", - "ad_click_actions", - "ad_id", - "ad_impression_actions", - "ad_name", - "adset_bid_value", - "adset_end", - "adset_id", - "adset_name", - "adset_start", - "age_targeting", - "attribution_setting", - "auction_bid", - "auction_competitiveness", - "auction_max_competitor_bid", - "buying_type", - "campaign_id", - "campaign_name", - "canvas_avg_view_percent", - "canvas_avg_view_time", - "catalog_segment_actions", - "catalog_segment_value", - "catalog_segment_value_mobile_purchase_roas", - "catalog_segment_value_omni_purchase_roas", - "catalog_segment_value_website_purchase_roas", - "clicks", - "conversion_rate_ranking", - "conversion_values", - "conversions", - "converted_product_quantity", - "converted_product_value", - "cost_per_15_sec_video_view", - "cost_per_2_sec_continuous_video_view", - "cost_per_action_type", - "cost_per_ad_click", - "cost_per_conversion", - "cost_per_dda_countby_convs", - "cost_per_estimated_ad_recallers", - "cost_per_inline_link_click", - "cost_per_inline_post_engagement", - "cost_per_one_thousand_ad_impression", - "cost_per_outbound_click", - "cost_per_thruplay", - "cost_per_unique_action_type", - "cost_per_unique_click", - "cost_per_unique_conversion", - "cost_per_unique_inline_link_click", - "cost_per_unique_outbound_click", - "cpc", - "cpm", - "cpp", - "created_time", - "ctr", - "date_start", - "date_stop", - "dda_countby_convs", - "dda_results", - "engagement_rate_ranking", - "estimated_ad_recall_rate", - "estimated_ad_recall_rate_lower_bound", - "estimated_ad_recall_rate_upper_bound", - "estimated_ad_recallers", - "estimated_ad_recallers_lower_bound", - "estimated_ad_recallers_upper_bound", - "frequency", - "full_view_impressions", - "full_view_reach", - "gender_targeting", - "impressions", - "inline_link_click_ctr", - "inline_link_clicks", - "inline_post_engagement", - "instant_experience_clicks_to_open", - "instant_experience_clicks_to_start", - "instant_experience_outbound_clicks", - "interactive_component_tap", - "labels", - "location", - "mobile_app_purchase_roas", - "objective", - "optimization_goal", - "outbound_clicks", - "outbound_clicks_ctr", - "place_page_name", - "purchase_roas", - "qualifying_question_qualify_answer_rate", - "quality_ranking", - "quality_score_ectr", - "quality_score_ecvr", - "quality_score_organic", - "reach", - "social_spend", - "spend", - "total_postbacks", - "unique_actions", - "unique_clicks", - "unique_conversions", - "unique_ctr", - "unique_inline_link_click_ctr", - "unique_inline_link_clicks", - "unique_link_clicks_ctr", - "unique_outbound_clicks", - "unique_outbound_clicks_ctr", - "unique_video_continuous_2_sec_watched_actions", - "unique_video_view_15_sec", - "updated_time", - "video_15_sec_watched_actions", - "video_30_sec_watched_actions", - "video_avg_time_watched_actions", - "video_continuous_2_sec_watched_actions", - "video_p100_watched_actions", - "video_p25_watched_actions", - "video_p50_watched_actions", - "video_p75_watched_actions", - "video_p95_watched_actions", - "video_play_actions", - "video_play_curve_actions", - "video_play_retention_0_to_15s_actions", - "video_play_retention_20_to_60s_actions", - "video_play_retention_graph_actions", - "video_thruplay_watched_actions", - "video_time_watched_actions", - "website_ctr", - "website_purchase_roas", - "wish_bid" - ] - } - }, - "breakdowns": { - "title": "Breakdowns", - "description": "A list of chosen breakdowns for breakdowns", - "default": [], - "type": "array", - "items": { - "title": "ValidBreakdowns", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", - "enum": [ - "ad_format_asset", - "age", - "app_id", - "body_asset", - "call_to_action_asset", - "country", - "description_asset", - "device_platform", - "dma", - "frequency_value", - "gender", - "hourly_stats_aggregated_by_advertiser_time_zone", - "hourly_stats_aggregated_by_audience_time_zone", - "image_asset", - "impression_device", - "link_url_asset", - "place_page_id", - "platform_position", - "product_id", - "publisher_platform", - "region", - "skan_conversion_id", - "title_asset", - "video_asset" - ] - } - }, - "action_breakdowns": { - "title": "Action Breakdowns", - "description": "A list of chosen action_breakdowns for action_breakdowns", - "default": [], - "type": "array", - "items": { - "title": "ValidActionBreakdowns", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", - "enum": [ - "action_canvas_component_name", - "action_carousel_card_id", - "action_carousel_card_name", - "action_destination", - "action_device", - "action_reaction", - "action_target_id", - "action_type", - "action_video_sound", - "action_video_type" - ] - } - }, - "time_increment": { - "title": "Time Increment", - "description": "Time window in days by which to aggregate statistics. The sync will be chunked into N day intervals, where N is the number of days you specified. For example, if you set this value to 7, then all statistics will be reported as 7-day aggregates by starting from the start_date. If the start and end dates are October 1st and October 30th, then the connector will output 5 records: 01 - 06, 07 - 13, 14 - 20, 21 - 27, and 28 - 30 (3 days only).", - "default": 1, - "exclusiveMaximum": 90, - "exclusiveMinimum": 0, - "type": "integer" - }, - "start_date": { - "title": "Start Date", - "description": "The date from which you'd like to replicate data for this stream, in the format YYYY-MM-DDT00:00:00Z.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for this stream, in the format YYYY-MM-DDT00:00:00Z. All data generated between the start date and this date will be replicated. Not setting this option will result in always syncing the latest data.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-26T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "insights_lookback_window": { - "title": "Custom Insights Lookback Window", - "description": "The attribution window", - "default": 28, - "maximum": 28, - "mininum": 1, - "exclusiveMinimum": 0, - "type": "integer" - } - }, - "required": ["name"] - } - }, - "page_size": { - "title": "Page Size of Requests", - "description": "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.", - "default": 100, - "order": 7, - "exclusiveMinimum": 0, - "type": "integer" - }, - "insights_lookback_window": { - "title": "Insights Lookback Window", - "description": "The attribution window", - "default": 28, - "order": 8, - "maximum": 28, - "mininum": 1, - "exclusiveMinimum": 0, - "type": "integer" - }, - "max_batch_size": { - "title": "Maximum size of Batched Requests", - "description": "Maximum batch size used when sending batch requests to Facebook API. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.", - "default": 50, - "order": 9, - "exclusiveMinimum": 0, - "type": "integer" - } - }, - "required": ["account_id", "start_date", "access_token"] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "dfd88b22-b603-4c3d-aad7-3701784586b1", - "name": "Faker", - "dockerRepository": "airbyte/source-faker", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/faker", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/faker", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Faker Source Spec", - "type": "object", - "required": ["count"], - "additionalProperties": false, - "properties": { - "count": { - "title": "Count", - "description": "How many users should be generated in total. This setting does not apply to the purchases or products stream.", - "type": "integer", - "minimum": 1, - "default": 1000, - "order": 0 - }, - "seed": { - "title": "Seed", - "description": "Manually control the faker random seed to return the same values on subsequent runs (leave -1 for random)", - "type": "integer", - "default": -1, - "order": 1 - }, - "records_per_sync": { - "title": "Records Per Sync", - "description": "How many fake records will be returned for each sync, for each stream? By default, it will take 2 syncs to create the requested 1000 records.", - "type": "integer", - "minimum": 1, - "default": 500, - "order": 2 - }, - "records_per_slice": { - "title": "Records Per Stream Slice", - "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", - "type": "integer", - "minimum": 1, - "default": 100, - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "778daa7c-feaf-4db6-96f3-70fd645acc77", - "name": "File", - "dockerRepository": "airbyte/source-file-secure", - "dockerImageTag": "0.2.20", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/file", - "icon": "file.svg", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/file", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "File Source Spec", - "type": "object", - "additionalProperties": true, - "required": ["dataset_name", "format", "url", "provider"], - "properties": { - "dataset_name": { - "type": "string", - "title": "Dataset Name", - "description": "The Name of the final table to replicate this file into (should include letters, numbers dash and underscores only)." - }, - "format": { - "type": "string", - "enum": [ - "csv", - "json", - "jsonl", - "excel", - "feather", - "parquet", - "yaml" - ], - "default": "csv", - "title": "File Format", - "description": "The Format of the file which should be replicated (Warning: some formats may be experimental, please refer to the docs)." - }, - "reader_options": { - "type": "string", - "title": "Reader Options", - "description": "This should be a string in JSON format. It depends on the chosen file format to provide additional options and tune its behavior.", - "examples": ["{}", "{\"sep\": \" \"}"] - }, - "url": { - "type": "string", - "title": "URL", - "description": "The URL path to access the file which should be replicated." - }, - "provider": { - "type": "object", - "title": "Storage Provider", - "description": "The storage Provider or Location of the file(s) which should be replicated.", - "default": "Public Web", - "oneOf": [ - { - "title": "HTTPS: Public Web", - "required": ["storage"], - "properties": { - "storage": { - "type": "string", - "const": "HTTPS" - }, - "user_agent": { - "type": "boolean", - "title": "User-Agent", - "default": false, - "description": "Add User-Agent to request" - } - } - }, - { - "title": "GCS: Google Cloud Storage", - "required": ["storage"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "GCS" - }, - "service_account_json": { - "type": "string", - "title": "Service Account JSON", - "description": "In order to access private Buckets stored on Google Cloud, this connector would need a service account json credentials with the proper permissions as described here. Please generate the credentials.json file and copy/paste its content to this field (expecting JSON formats). If accessing publicly available data, this field is not necessary." - } - } - }, - { - "title": "S3: Amazon Web Services", - "required": ["storage"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "S3" - }, - "aws_access_key_id": { - "type": "string", - "title": "AWS Access Key ID", - "description": "In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary." - }, - "aws_secret_access_key": { - "type": "string", - "title": "AWS Secret Access Key", - "description": "In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true - } - } - }, - { - "title": "AzBlob: Azure Blob Storage", - "required": ["storage", "storage_account"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "AzBlob" - }, - "storage_account": { - "type": "string", - "title": "Storage Account", - "description": "The globally unique name of the storage account that the desired blob sits within. See here for more details." - }, - "sas_token": { - "type": "string", - "title": "SAS Token", - "description": "To access Azure Blob Storage, this connector would need credentials with the proper permissions. One option is a SAS (Shared Access Signature) token. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true - }, - "shared_key": { - "type": "string", - "title": "Shared Key", - "description": "To access Azure Blob Storage, this connector would need credentials with the proper permissions. One option is a storage account shared key (aka account key or access key). If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true - } - } - }, - { - "title": "SSH: Secure Shell", - "required": ["storage", "user", "host"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "SSH" - }, - "user": { - "type": "string", - "title": "User", - "description": "" - }, - "password": { - "type": "string", - "title": "Password", - "description": "", - "airbyte_secret": true - }, - "host": { - "type": "string", - "title": "Host", - "description": "" - }, - "port": { - "type": "string", - "title": "Port", - "default": "22", - "description": "" - } - } - }, - { - "title": "SCP: Secure copy protocol", - "required": ["storage", "user", "host"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "SCP" - }, - "user": { - "type": "string", - "title": "User", - "description": "" - }, - "password": { - "type": "string", - "title": "Password", - "description": "", - "airbyte_secret": true - }, - "host": { - "type": "string", - "title": "Host", - "description": "" - }, - "port": { - "type": "string", - "title": "Port", - "default": "22", - "description": "" - } - } - }, - { - "title": "SFTP: Secure File Transfer Protocol", - "required": ["storage", "user", "host"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "SFTP" - }, - "user": { - "type": "string", - "title": "User", - "description": "" - }, - "password": { - "type": "string", - "title": "Password", - "description": "", - "airbyte_secret": true - }, - "host": { - "type": "string", - "title": "Host", - "description": "" - }, - "port": { - "type": "string", - "title": "Port", - "default": "22", - "description": "" - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "6f2ac653-8623-43c4-8950-19218c7caf3d", - "name": "Firebolt", - "dockerRepository": "airbyte/source-firebolt", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/firebolt", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/firebolt", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Firebolt Spec", - "type": "object", - "required": ["username", "password", "database"], - "additionalProperties": false, - "properties": { - "username": { - "type": "string", - "title": "Username", - "description": "Firebolt email address you use to login.", - "examples": ["username@email.com"] - }, - "password": { - "type": "string", - "title": "Password", - "description": "Firebolt password." - }, - "account": { - "type": "string", - "title": "Account", - "description": "Firebolt account to login." - }, - "host": { - "type": "string", - "title": "Host", - "description": "The host name of your Firebolt database.", - "examples": ["api.app.firebolt.io"] - }, - "database": { - "type": "string", - "title": "Database", - "description": "The database to connect to." - }, - "engine": { - "type": "string", - "title": "Engine", - "description": "Engine name or url to connect to." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567", - "name": "Freshdesk", - "dockerRepository": "airbyte/source-freshdesk", - "dockerImageTag": "0.3.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshdesk", - "icon": "freshdesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshdesk", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Freshdesk Spec", - "type": "object", - "required": ["domain", "api_key"], - "additionalProperties": true, - "properties": { - "domain": { - "type": "string", - "description": "Freshdesk domain", - "title": "Domain", - "examples": ["myaccount.freshdesk.com"], - "pattern": "^[a-zA-Z0-9._-]*\\.freshdesk\\.com$" - }, - "api_key": { - "type": "string", - "title": "API Key", - "description": "Freshdesk API Key. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, - "requests_per_minute": { - "title": "Requests per minute", - "type": "integer", - "description": "The number of requests per minute that this source allowed to use. There is a rate limit of 50 requests per minute per app per account." - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "UTC date and time. Any data created after this date will be replicated. If this parameter is not set, all data will be replicated.", - "format": "date-time", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2020-12-01T00:00:00Z"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", - "name": "GitHub", - "dockerRepository": "airbyte/source-github", - "dockerImageTag": "0.2.46", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/github", - "icon": "github.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/github", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "GitHub Source Spec", - "type": "object", - "required": ["start_date", "repository"], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authentication *", - "description": "Choose how to authenticate to GitHub", - "type": "object", - "order": 0, - "oneOf": [ - { - "type": "object", - "title": "OAuth", - "required": ["access_token"], - "properties": { - "option_title": { - "type": "string", - "const": "OAuth Credentials", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "OAuth access token", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Personal Access Token", - "required": ["personal_access_token"], - "properties": { - "option_title": { - "type": "string", - "const": "PAT Credentials", - "order": 0 - }, - "personal_access_token": { - "type": "string", - "title": "Personal Access Tokens", - "description": "Log into GitHub and then generate a personal access token. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with \",\"", - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "type": "string", - "title": "Start date", - "description": "The date from which you'd like to replicate data from GitHub in the format YYYY-MM-DDT00:00:00Z. For the streams which support this configuration, only data generated on or after the start date will be replicated. This field doesn't apply to all streams, see the docs for more info", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 1 - }, - "repository": { - "type": "string", - "examples": [ - "airbytehq/airbyte airbytehq/another-repo", - "airbytehq/*", - "airbytehq/airbyte" - ], - "title": "GitHub Repositories", - "description": "Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.", - "order": 2 - }, - "branch": { - "type": "string", - "title": "Branch (Optional)", - "examples": [ - "airbytehq/airbyte/master airbytehq/airbyte/my-branch" - ], - "description": "Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.", - "order": 3 - }, - "page_size_for_large_streams": { - "type": "integer", - "title": "Page size for large streams (Optional)", - "minimum": 1, - "maximum": 100, - "default": 10, - "description": "The Github connector contains several streams with a large amount of data. The page size of such streams depends on the size of your repository. We recommended that you specify values between 10 and 30.", - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "option_title"], - "predicate_value": "OAuth Credentials", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "5e6175e5-68e1-4c17-bff9-56103bbb0d80", - "name": "Gitlab", - "dockerRepository": "airbyte/source-gitlab", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab", - "icon": "gitlab.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Gitlab Singer Spec", - "type": "object", - "required": ["api_url", "private_token", "start_date"], - "additionalProperties": false, - "properties": { - "api_url": { - "type": "string", - "examples": ["gitlab.com"], - "description": "Please enter your basic URL from Gitlab instance", - "pattern": "^gitlab[a-zA-Z0-9._-]*\\.com$" - }, - "private_token": { - "type": "string", - "description": "Log into your Gitlab account and then generate a personal Access Token.", - "airbyte_secret": true - }, - "groups": { - "type": "string", - "examples": ["airbyte.io"], - "description": "Space-delimited list of groups. e.g. airbyte.io" - }, - "projects": { - "type": "string", - "examples": ["airbyte.io/documentation"], - "description": "Space-delimited list of projects. e.g. airbyte.io/documentation meltano/tap-gitlab" - }, - "start_date": { - "type": "string", - "description": "The date from which you'd like to replicate data for Gitlab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "253487c0-2246-43ba-a21f-5116b20a2c50", - "name": "Google Ads", - "dockerRepository": "airbyte/source-google-ads", - "dockerImageTag": "0.1.44", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-ads", - "icon": "google-adwords.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-ads", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Ads Spec", - "type": "object", - "required": ["credentials", "start_date", "customer_id"], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "object", - "description": "", - "title": "Google Credentials", - "order": 0, - "required": [ - "developer_token", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "developer_token": { - "type": "string", - "title": "Developer Token", - "order": 0, - "description": "Developer token granted by Google to use their APIs. More instruction on how to find this value in our docs", - "airbyte_secret": true - }, - "client_id": { - "type": "string", - "title": "Client ID", - "order": 1, - "description": "The Client ID of your Google Ads developer application. More instruction on how to find this value in our docs" - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "order": 2, - "description": "The Client Secret of your Google Ads developer application. More instruction on how to find this value in our docs", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "order": 3, - "description": "The token for obtaining a new access token. More instruction on how to find this value in our docs", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token (Optional)", - "order": 4, - "description": "Access Token for making authenticated requests. More instruction on how to find this value in our docs", - "airbyte_secret": true - } - } - }, - "customer_id": { - "title": "Customer ID(s)", - "type": "string", - "description": "Comma separated list of (client) customer IDs. Each customer ID must be specified as a 10-digit number without dashes. More instruction on how to find this value in our docs. Metrics streams like AdGroupAdReport cannot be requested for a manager account.", - "pattern": "^[0-9]{10}(,[0-9]{10})*$", - "examples": ["6783948572,5839201945"], - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2017-01-25"], - "order": 2 - }, - "end_date": { - "type": "string", - "title": "End Date (Optional)", - "description": "UTC date and time in the format 2017-01-25. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2017-01-30"], - "order": 6 - }, - "custom_queries": { - "type": "array", - "title": "Custom GAQL Queries (Optional)", - "description": "", - "order": 3, - "items": { - "type": "object", - "properties": { - "query": { - "type": "string", - "title": "Custom Query", - "description": "A custom defined GAQL query for building the report. Should not contain segments.date expression because it is used by incremental streams. See Google's query builder for more information.", - "examples": [ - "SELECT segments.ad_destination_type, campaign.advertising_channel_sub_type FROM campaign WHERE campaign.status = 'PAUSED'" - ] - }, - "table_name": { - "type": "string", - "title": "Destination Table Name", - "description": "The table name in your destination database for choosen query." - } - } - } - }, - "login_customer_id": { - "type": "string", - "title": "Login Customer ID for Managed Accounts (Optional)", - "description": "If your access to the customer account is through a manager account, this field is required and must be set to the customer ID of the manager account (10-digit number without dashes). More information about this field you can see here", - "pattern": "^([0-9]{10})?$", - "examples": ["7349206847"], - "order": 4 - }, - "conversion_window_days": { - "title": "Conversion Window (Optional)", - "type": "integer", - "description": "A conversion window is the period of time after an ad interaction (such as an ad click or video view) during which a conversion, such as a purchase, is recorded in Google Ads. For more information, see Google's documentation.", - "minimum": 0, - "maximum": 1095, - "default": 14, - "examples": [14], - "order": 5 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [ - ["client_id"], - ["client_secret"], - ["developer_token"] - ], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003", - "name": "Google Analytics", - "dockerRepository": "airbyte/source-google-analytics-v4", - "dockerImageTag": "0.1.25", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics", - "icon": "google-analytics.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Analytics V4 Spec", - "type": "object", - "required": ["view_id", "start_date"], - "additionalProperties": true, - "properties": { - "credentials": { - "order": 0, - "type": "object", - "title": "Credentials", - "description": "Credentials for the service", - "oneOf": [ - { - "title": "Authenticate via Google (Oauth)", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Google Analytics developer application.", - "airbyte_secret": true, - "order": 1 - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Google Analytics developer application.", - "airbyte_secret": true, - "order": 2 - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining a new access token.", - "airbyte_secret": true, - "order": 3 - }, - "access_token": { - "title": "Access Token (Optional)", - "type": "string", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "type": "object", - "title": "Service Account Key Authentication", - "required": ["credentials_json"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service", - "order": 0 - }, - "credentials_json": { - "title": "Service Account JSON Key", - "type": "string", - "description": "The JSON key of the service account to use for authorization", - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ], - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "order": 1, - "type": "string", - "title": "Replication Start Date", - "description": "The date in the format YYYY-MM-DD. Any data before this date will not be replicated.", - "examples": ["2020-06-01"] - }, - "view_id": { - "order": 2, - "type": "string", - "title": "View ID", - "description": "The ID for the Google Analytics View you want to fetch data from. This can be found from the Google Analytics Account Explorer." - }, - "custom_reports": { - "order": 3, - "type": "string", - "title": "Custom Reports (Optional)", - "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the docs for more information about the exact format you can use to fill out this field." - }, - "window_in_days": { - "type": "integer", - "title": "Data request time increment in days (Optional)", - "description": "The time increment used by the connector when requesting data from the Google Analytics API. More information is available in the the docs. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. ", - "examples": [30, 60, 90, 120, 200, 364], - "default": 1, - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "d19ae824-e289-4b14-995a-0632eb46d246", - "name": "Google Directory", - "dockerRepository": "airbyte/source-google-directory", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-directory", - "icon": "googledirectory.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-directory", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Directory Spec", - "type": "object", - "required": ["credentials_json", "email"], - "additionalProperties": false, - "properties": { - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. See the docs for more information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "eb4c9e00-db83-4d63-a386-39cfa91012a8", - "name": "Google Search Console", - "dockerRepository": "airbyte/source-google-search-console", - "dockerImageTag": "0.1.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-search-console", - "icon": "googlesearchconsole.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-search-console", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Search Console Spec", - "type": "object", - "required": ["site_urls", "start_date", "authorization"], - "properties": { - "site_urls": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Website URL Property", - "description": "The URLs of the website property attached to your GSC account. Read more here.", - "examples": ["https://example1.com", "https://example2.com"], - "order": 0 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date in the format 2017-01-25. Any data before this date will not be replicated.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 1 - }, - "end_date": { - "type": "string", - "title": "End Date", - "description": "UTC date in the format 2017-01-25. Any data after this date will not be replicated. Must be greater or equal to the start date field.", - "examples": ["2021-12-12"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 2 - }, - "authorization": { - "type": "object", - "title": "Authentication Type", - "description": "", - "order": 3, - "oneOf": [ - { - "title": "OAuth", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The client ID of your Google Search Console developer application. Read more here.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The client secret of your Google Search Console developer application. Read more here.", - "airbyte_secret": true - }, - "access_token": { - "title": "Access Token", - "type": "string", - "description": "Access token for making authenticated requests. Read more here.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining a new access token. Read more here.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Service Account Key Authentication", - "required": ["auth_type", "service_account_info", "email"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service", - "order": 0 - }, - "service_account_info": { - "title": "Service Account JSON Key", - "type": "string", - "description": "The JSON key of the service account to use for authorization. Read more here.", - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ] - }, - "email": { - "title": "Admin Email", - "type": "string", - "description": "The email of the user which has permissions to access the Google Workspace Admin APIs." - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["authorization", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "71607ba1-c0ac-4799-8049-7f4b90dd50f7", - "name": "Google Sheets", - "dockerRepository": "airbyte/source-google-sheets", - "dockerImageTag": "0.2.17", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-sheets", - "icon": "google-sheets.svg", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-sheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["spreadsheet_id", "credentials"], - "additionalProperties": true, - "properties": { - "spreadsheet_id": { - "type": "string", - "title": "Spreadsheet Link", - "description": "Enter the link to the Google spreadsheet you want to sync", - "examples": [ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit" - ] - }, - "row_batch_size": { - "type": "integer", - "title": "Row Batch Size", - "description": "Number of rows fetched when making a Google Sheet API call. Defaults to 200.", - "default": 200 - }, - "credentials": { - "type": "object", - "title": "Authentication", - "description": "Credentials for connecting to the Google Sheets API", - "oneOf": [ - { - "title": "Authenticate via Google (OAuth)", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "Enter your Google application's Client ID", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "Enter your Google application's Client Secret", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "Enter your Google application's refresh token", - "airbyte_secret": true - } - } - }, - { - "title": "Service Account Key Authentication", - "type": "object", - "required": ["auth_type", "service_account_info"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service" - }, - "service_account_info": { - "type": "string", - "title": "Service Account Information.", - "description": "Enter your Google Cloud service account key in JSON format", - "airbyte_secret": true, - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ] - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734", - "name": "Google Workspace Admin Reports", - "dockerRepository": "airbyte/source-google-workspace-admin-reports", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports", - "icon": "googleworkpace.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Directory Spec", - "type": "object", - "required": ["credentials_json", "email"], - "additionalProperties": false, - "properties": { - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. See the docs for more information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs." - }, - "lookback": { - "type": "integer", - "minimum": 0, - "maximum": 180, - "description": "Sets the range of time shown in the report. Reports API allows from up to 180 days ago. " - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44", - "name": "Greenhouse", - "dockerRepository": "airbyte/source-greenhouse", - "dockerImageTag": "0.2.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse", - "icon": "greenhouse.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Greenhouse Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "Greenhouse API Key. See the docs for more information on how to generate this key.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", - "name": "HubSpot", - "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.82", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", - "icon": "hubspot.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "HubSpot Source Spec", - "type": "object", - "required": ["start_date", "credentials"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"] - }, - "credentials": { - "title": "Authentication", - "description": "Choose how to authenticate to HubSpot.", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "credentials_title" - ], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials", - "description": "Name of the credentials", - "const": "OAuth Credentials", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "description": "The Client ID of your HubSpot developer application. See the Hubspot docs if you need help finding this ID.", - "type": "string", - "examples": ["123456789000"] - }, - "client_secret": { - "title": "Client Secret", - "description": "The client secret for your HubSpot developer application. See the Hubspot docs if you need help finding this secret.", - "type": "string", - "examples": ["secret"], - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Refresh token to renew an expired access token. See the Hubspot docs if you need help finding this token.", - "type": "string", - "examples": ["refresh_token"], - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API key", - "required": ["api_key", "credentials_title"], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials", - "description": "Name of the credentials set", - "const": "API Key Credentials", - "order": 0 - }, - "api_key": { - "title": "API key", - "description": "HubSpot API Key. See the Hubspot docs if you need help finding this key.", - "type": "string", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Private APP", - "required": ["access_token", "credentials_title"], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials", - "description": "Name of the credentials set", - "const": "Private App Credentials", - "order": 0 - }, - "access_token": { - "title": "Access token", - "description": "HubSpot Access token. See the Hubspot docs if you need help finding this token.", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "6acf6b55-4f1e-4fca-944e-1a3caef8aba8", - "name": "Instagram", - "dockerRepository": "airbyte/source-instagram", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/instagram", - "icon": "instagram.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/instagram", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/instagram", - "connectionSpecification": { - "title": "Source Instagram", - "type": "object", - "properties": { - "start_date": { - "title": "Start Date", - "description": "The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "access_token": { - "title": "Access Token", - "description": "The value of the access token generated. See the docs for more information", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["start_date", "access_token"] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", - "name": "Intercom", - "dockerRepository": "airbyte/source-intercom", - "dockerImageTag": "0.1.25", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", - "icon": "intercom.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Intercom Spec", - "type": "object", - "required": ["start_date", "access_token"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2020-11-16T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "access_token": { - "title": "Access token", - "type": "string", - "description": "Access token for making authenticated requests. See the Intercom docs for more information.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "2e875208-0c0b-4ee4-9e92-1cb3156ea799", - "name": "Iterable", - "dockerRepository": "airbyte/source-iterable", - "dockerImageTag": "0.1.15", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/iterable", - "icon": "iterable.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/iterable", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Iterable Spec", - "type": "object", - "required": ["start_date", "api_key"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2021-04-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "api_key": { - "type": "string", - "title": "API Key", - "description": "Iterable API Key. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "95e8cffd-b8c4-4039-968e-d32fb4a69bde", - "name": "Klaviyo", - "dockerRepository": "airbyte/source-klaviyo", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/klaviyo", - "icon": "klaviyo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/klaviyo", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/klaviyo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Klaviyo Spec", - "type": "object", - "properties": { - "api_key": { - "title": "Api Key", - "description": "Klaviyo API Key. See our docs if you need help finding this key.", - "airbyte_secret": true, - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string" - } - }, - "required": ["api_key", "start_date"] - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "cd06e646-31bf-4dc8-af48-cbc6530fcad3", - "name": "Kustomer", - "dockerRepository": "airbyte/source-kustomer-singer", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/kustomer", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/kustomer", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Kustomer Singer Spec", - "type": "object", - "required": ["api_token", "start_date"], - "additionalProperties": true, - "properties": { - "api_token": { - "title": "API Token", - "type": "string", - "description": "Kustomer API Token. See the docs on how to obtain this", - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate the data", - "examples": ["2019-01-01T00:00:00Z"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "789f8e7a-2d28-11ec-8d3d-0242ac130003", - "name": "Lemlist", - "dockerRepository": "airbyte/source-lemlist", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/lemlist", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Lemlist Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "API key to access your lemlist account.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "137ece28-5434-455c-8f34-69dc3782f451", - "name": "LinkedIn Ads", - "dockerRepository": "airbyte/source-linkedin-ads", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads", - "icon": "linkedin.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Linkedin Ads Spec", - "type": "object", - "required": ["start_date"], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authentication *", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "auth_method": { - "type": "string", - "const": "oAuth2.0" - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The client ID of the LinkedIn Ads developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client secret", - "description": "The client secret the LinkedIn Ads developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh token", - "description": "The key to refresh the expired access token.", - "airbyte_secret": true - } - } - }, - { - "title": "Access token", - "type": "object", - "required": ["access_token"], - "properties": { - "auth_method": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - "title": "Access token", - "description": "The token value generated using the authentication code. See the docs to obtain yours.", - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "type": "string", - "title": "Start date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "description": "UTC date in the format 2020-09-17. Any data before this date will not be replicated.", - "examples": ["2021-05-17"] - }, - "account_ids": { - "title": "Account IDs (Optional)", - "type": "array", - "description": "Specify the account IDs separated by a space, to pull the data from. Leave empty, if you want to pull the data from all associated accounts. See the LinkedIn Ads docs for more info.", - "items": { - "type": "integer" - }, - "default": [] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e", - "name": "Linnworks", - "dockerRepository": "airbyte/source-linnworks", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linnworks", - "icon": "linnworks.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linnworks", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Linnworks Spec", - "type": "object", - "required": [ - "application_id", - "application_secret", - "token", - "start_date" - ], - "additionalProperties": false, - "properties": { - "application_id": { - "title": "Application ID.", - "description": "Linnworks Application ID", - "type": "string" - }, - "application_secret": { - "title": "Application Secret", - "description": "Linnworks Application Secret", - "type": "string", - "airbyte_secret": true - }, - "token": { - "title": "API Token", - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "type": "string", - "format": "date-time" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", - "name": "Looker", - "dockerRepository": "airbyte/source-looker", - "dockerImageTag": "0.2.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/looker", - "icon": "looker.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/looker", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Looker Spec", - "type": "object", - "required": ["domain", "client_id", "client_secret"], - "additionalProperties": false, - "properties": { - "domain": { - "type": "string", - "title": "Domain", - "examples": [ - "domainname.looker.com", - "looker.clientname.com", - "123.123.124.123:8000" - ], - "description": "Domain for your Looker account, e.g. airbyte.cloud.looker.com,looker.[clientname].com,IP address" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID is first part of an API3 key that is specific to each Looker user. See the docs for more information on how to generate this key." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret is second part of an API3 key." - }, - "run_look_ids": { - "title": "Look IDs to Run", - "type": "array", - "items": { - "type": "string", - "pattern": "^[0-9]*$" - }, - "description": "The IDs of any Looks to run (optional)" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b03a9f3e-22a5-11eb-adc1-0242ac120002", - "name": "Mailchimp", - "dockerRepository": "airbyte/source-mailchimp", - "dockerImageTag": "0.2.14", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mailchimp", - "icon": "mailchimp.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mailchimp", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Mailchimp Spec", - "type": "object", - "required": [], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "object", - "title": "Authentication *", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "required": ["auth_type", "access_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "title": "Access Token", - "type": "string", - "description": "An access token generated using the above client ID and secret.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Key", - "required": ["auth_type", "apikey"], - "properties": { - "auth_type": { - "type": "string", - "const": "apikey", - "order": 1 - }, - "apikey": { - "type": "string", - "title": "API Key", - "description": "Mailchimp API Key. See the docs for information on how to generate this key.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348", - "name": "Marketo", - "dockerRepository": "airbyte/source-marketo", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/marketo", - "icon": "marketo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/marketo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Marketo Spec", - "type": "object", - "required": [ - "domain_url", - "client_id", - "client_secret", - "start_date" - ], - "additionalProperties": true, - "properties": { - "domain_url": { - "title": "Domain URL", - "type": "string", - "order": 3, - "description": "Your Marketo Base URL. See the docs for info on how to obtain this.", - "examples": ["https://000-AAA-000.mktorest.com"], - "airbyte_secret": true - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Marketo developer application. See the docs for info on how to obtain this.", - "order": 0, - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Marketo developer application. See the docs for info on how to obtain this.", - "order": 1, - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "type": "string", - "order": 2, - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2020-09-25T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "c7cb421b-942e-4468-99ee-e369bcabaec5", - "name": "Metabase", - "dockerRepository": "airbyte/source-metabase", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/metabase", - "icon": "metabase.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/metabase", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Metabase Source Spec", - "type": "object", - "required": ["instance_api_url"], - "additionalProperties": true, - "properties": { - "instance_api_url": { - "type": "string", - "title": "Metabase Instance API URL", - "description": "URL to your metabase instance API", - "examples": ["http://localhost:3000/api/"], - "order": 0 - }, - "username": { - "type": "string", - "order": 1 - }, - "password": { - "type": "string", - "airbyte_secret": true, - "order": 2 - }, - "session_token": { - "type": "string", - "description": "To generate your session token, you need to run the following command: ``` curl -X POST \\\n -H \"Content-Type: application/json\" \\\n -d '{\"username\": \"person@metabase.com\", \"password\": \"fakepassword\"}' \\\n http://localhost:3000/api/session\n``` Then copy the value of the `id` field returned by a successful call to that API.\nNote that by default, sessions are good for 14 days and needs to be regenerated.", - "airbyte_secret": true, - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", - "name": "Microsoft SQL Server (MSSQL)", - "dockerRepository": "airbyte/source-mssql-strict-encrypt", - "dockerImageTag": "0.4.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mssql", - "icon": "mssql.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mssql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MSSQL Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "description": "The hostname of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port of the database.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "examples": ["1433"], - "order": 1 - }, - "database": { - "description": "The name of the database.", - "title": "Database", - "type": "string", - "examples": ["master"], - "order": 2 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 3 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 5 - }, - "ssl_method": { - "title": "SSL Method", - "type": "object", - "description": "The encryption method which is used when communicating with the database.", - "order": 6, - "oneOf": [ - { - "title": "Encrypted (trust server certificate)", - "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", - "required": ["ssl_method"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_trust_server_certificate", - "enum": ["encrypted_trust_server_certificate"], - "default": "encrypted_trust_server_certificate" - } - } - }, - { - "title": "Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": [ - "ssl_method", - "trustStoreName", - "trustStorePassword" - ], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" - }, - "hostNameInCertificate": { - "title": "Host Name In Certificate", - "type": "string", - "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 - } - } - } - ] - }, - "replication": { - "type": "object", - "title": "Replication Method", - "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "default": "STANDARD", - "order": 8, - "oneOf": [ - { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["replication_type"], - "properties": { - "replication_type": { - "type": "string", - "const": "STANDARD", - "enum": ["STANDARD"], - "default": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "required": ["replication_type"], - "properties": { - "replication_type": { - "type": "string", - "const": "CDC", - "enum": ["CDC"], - "default": "CDC", - "order": 0 - }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "eaf50f04-21dd-4620-913b-2a83f5635227", - "name": "Microsoft teams", - "dockerRepository": "airbyte/source-microsoft-teams", - "dockerImageTag": "0.2.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/microsoft-teams", - "icon": "microsoft-teams.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/microsoft-teams", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Microsoft Teams Spec", - "type": "object", - "required": ["period"], - "additionalProperties": true, - "properties": { - "period": { - "type": "string", - "title": "Period", - "description": "Specifies the length of time over which the Team Device Report stream is aggregated. The supported values are: D7, D30, D90, and D180.", - "examples": ["D7"] - }, - "credentials": { - "title": "Authentication mechanism", - "description": "Choose how to authenticate to Microsoft", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "Authenticate via Microsoft (OAuth 2.0)", - "required": [ - "tenant_id", - "client_id", - "client_secret", - "refresh_token" - ], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "enum": ["Client"], - "default": "Client", - "order": 0 - }, - "tenant_id": { - "title": "Directory (tenant) ID", - "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Microsoft Teams developer application." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Microsoft Teams developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "A Refresh Token to renew the expired Access Token.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Authenticate via Microsoft", - "required": ["tenant_id", "client_id", "client_secret"], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Token", - "enum": ["Token"], - "default": "Token", - "order": 0 - }, - "tenant_id": { - "title": "Directory (tenant) ID", - "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Microsoft Teams developer application." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Microsoft Teams developer application.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "tenant_id": { - "type": "string", - "path_in_connector_config": ["credentials", "tenant_id"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a", - "name": "Mixpanel", - "dockerRepository": "airbyte/source-mixpanel", - "dockerImageTag": "0.1.20", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mixpanel", - "icon": "mixpanel.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mixpanel", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Mixpanel Spec", - "type": "object", - "required": ["api_secret"], - "properties": { - "api_secret": { - "order": 0, - "title": "Project Secret", - "type": "string", - "description": "Mixpanel project secret. See the docs for more information on how to obtain this.", - "airbyte_secret": true - }, - "attribution_window": { - "order": 1, - "title": "Attribution Window", - "type": "integer", - "description": " A period of time for attributing results to ads and the lookback period after those actions occur during which ad results are counted. Default attribution window is 5 days.", - "default": 5 - }, - "project_timezone": { - "order": 2, - "title": "Project Timezone", - "type": "string", - "description": "Time zone in which integer date times are stored. The project timezone may be found in the project settings in the Mixpanel console.", - "default": "US/Pacific", - "examples": ["US/Pacific", "UTC"] - }, - "select_properties_by_default": { - "order": 3, - "title": "Select Properties By Default", - "type": "boolean", - "description": "Setting this config parameter to TRUE ensures that new properties on events and engage records are captured. Otherwise new properties will be ignored.", - "default": true - }, - "start_date": { - "order": 4, - "title": "Start Date", - "type": "string", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If this option is not set, the connector will replicate data from up to one year ago by default.", - "examples": ["2021-11-16"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" - }, - "end_date": { - "order": 5, - "title": "End Date", - "type": "string", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated. Left empty to always sync to most recent date", - "examples": ["2021-11-16"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" - }, - "region": { - "order": 6, - "title": "Region", - "description": "The region of mixpanel domain instance either US or EU.", - "type": "string", - "enum": ["US", "EU"], - "default": "US" - }, - "date_window_size": { - "order": 7, - "title": "Date slicing window", - "description": "Defines window size in days, that used to slice through data. You can reduce it, if amount of data in each window is too big for your environment.", - "type": "integer", - "default": 30 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "80a54ea2-9959-4040-aac1-eee42423ec9b", - "name": "Monday", - "dockerRepository": "airbyte/source-monday", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/monday", - "icon": "monday.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/monday", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Monday Spec", - "type": "object", - "required": [], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_type", - "client_id", - "client_secret", - "access_token" - ], - "properties": { - "subdomain": { - "type": "string", - "title": "Subdomain/Slug (Optional)", - "description": "Slug/subdomain of the account, or the first part of the URL that comes before .monday.com", - "default": "", - "order": 0 - }, - "auth_type": { - "type": "string", - "const": "oauth2.0", - "order": 1 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_type", "api_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "api_token", - "order": 0 - }, - "api_token": { - "type": "string", - "title": "Personal API Token", - "description": "API Token for making authenticated requests.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["credentials", "subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e", - "name": "MongoDb", - "dockerRepository": "airbyte/source-mongodb-strict-encrypt", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mongodb-v2", - "icon": "mongodb.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mongodb-v2", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/mongodb-v2", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MongoDb Source Spec", - "type": "object", - "required": ["database"], - "additionalProperties": true, - "properties": { - "instance_type": { - "type": "object", - "title": "MongoDb Instance Type", - "description": "The MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", - "order": 0, - "oneOf": [ - { - "title": "Standalone MongoDb Instance", - "required": ["instance", "host", "port"], - "properties": { - "instance": { - "type": "string", - "enum": ["standalone"], - "default": "standalone" - }, - "host": { - "title": "Host", - "type": "string", - "description": "The host name of the Mongo database.", - "order": 0 - }, - "port": { - "title": "Port", - "type": "integer", - "description": "The port of the Mongo database.", - "minimum": 0, - "maximum": 65536, - "default": 27017, - "examples": ["27017"], - "order": 1 - } - } - }, - { - "title": "Replica Set", - "required": ["instance", "server_addresses"], - "properties": { - "instance": { - "type": "string", - "enum": ["replica"], - "default": "replica" - }, - "server_addresses": { - "title": "Server Addresses", - "type": "string", - "description": "The members of a replica set. Please specify `host`:`port` of each member separated by comma.", - "examples": ["host1:27017,host2:27017,host3:27017"], - "order": 0 - }, - "replica_set": { - "title": "Replica Set", - "type": "string", - "description": "A replica set in MongoDB is a group of mongod processes that maintain the same data set.", - "order": 1 - } - } - }, - { - "title": "MongoDB Atlas", - "additionalProperties": false, - "required": ["instance", "cluster_url"], - "properties": { - "instance": { - "type": "string", - "enum": ["atlas"], - "default": "atlas" - }, - "cluster_url": { - "title": "Cluster URL", - "type": "string", - "description": "The URL of a cluster to connect to.", - "order": 0 - } - } - } - ] - }, - "database": { - "title": "Database Name", - "type": "string", - "description": "The database you want to replicate.", - "order": 1 - }, - "user": { - "title": "User", - "type": "string", - "description": "The username which is used to access the database.", - "order": 2 - }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated with this username.", - "airbyte_secret": true, - "order": 3 - }, - "auth_source": { - "title": "Authentication Source", - "type": "string", - "description": "The authentication source where the user information is stored.", - "default": "admin", - "examples": ["admin"], - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "722ba4bf-06ec-45a4-8dd5-72e4a5cf3903", - "name": "My Hours", - "dockerRepository": "airbyte/source-my-hours", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/my-hours", - "icon": "my-hours.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/my-hours", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "My Hours Spec", - "type": "object", - "required": ["email", "password", "start_date"], - "additionalProperties": false, - "properties": { - "email": { - "title": "Email", - "type": "string", - "description": "Your My Hours username", - "example": "john@doe.com" - }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated to the username", - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "description": "Start date for collecting time logs", - "examples": ["%Y-%m-%d", "2016-01-01"], - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "logs_batch_size": { - "title": "Time logs batch size", - "description": "Pagination size used for retrieving logs in days", - "examples": [30], - "type": "integer", - "minimum": 1, - "maximum": 365, - "default": 30 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", - "name": "MySQL", - "dockerRepository": "airbyte/source-mysql-strict-encrypt", - "dockerImageTag": "0.6.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql", - "icon": "mysql.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MySql Source Spec", - "type": "object", - "required": [ - "host", - "port", - "database", - "username", - "replication_method" - ], - "properties": { - "host": { - "description": "The host name of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port to connect to.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "description": "The database name.", - "title": "Database", - "type": "string", - "order": 2 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 3 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 5 - }, - "replication_method": { - "type": "string", - "title": "Replication Method", - "description": "Replication method which is used for data extraction from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "order": 7, - "default": "STANDARD", - "enum": ["STANDARD", "CDC"] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "1d4fdb25-64fc-4569-92da-fcdca79a8372", - "name": "Okta", - "dockerRepository": "airbyte/source-okta", - "dockerImageTag": "0.1.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/okta", - "icon": "okta.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/okta", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Okta Spec", - "type": "object", - "required": [], - "additionalProperties": true, - "properties": { - "domain": { - "type": "string", - "title": "Okta domain", - "description": "The Okta domain. See the docs for instructions on how to find it.", - "airbyte_secret": false - }, - "start_date": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format YYYY-MM-DDTHH:MM:SSZ. Any data before this date will not be replicated.", - "examples": ["2022-07-22T00:00:00Z"], - "title": "Start Date" - }, - "credentials": { - "title": "Authorization Method *", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to obtain new Access Token, when it's expired.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_type", "api_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "api_token", - "order": 0 - }, - "api_token": { - "type": "string", - "title": "Personal API Token", - "description": "An Okta token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["domain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83", - "name": "Oracle DB", - "dockerRepository": "airbyte/source-oracle-strict-encrypt", - "dockerImageTag": "0.3.17", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle", - "icon": "oracle.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Oracle Source Spec", - "type": "object", - "required": ["host", "port", "username", "encryption"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 1 - }, - "port": { - "title": "Port", - "description": "Port of the database.\nOracle Corporations recommends the following port numbers:\n1521 - Default listening port for client connections to the listener. \n2484 - Recommended and officially registered listening port for client connections to the listener using TCP/IP with SSL", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 1521, - "order": 2 - }, - "connection_data": { - "title": "Connect by", - "type": "object", - "description": "Connect data that will be used for DB connection", - "order": 3, - "oneOf": [ - { - "title": "Service name", - "description": "Use service name", - "required": ["service_name"], - "properties": { - "connection_type": { - "type": "string", - "const": "service_name", - "default": "service_name", - "order": 0 - }, - "service_name": { - "title": "Service name", - "type": "string", - "order": 1 - } - } - }, - { - "title": "System ID (SID)", - "description": "Use SID (Oracle System Identifier)", - "required": ["sid"], - "properties": { - "connection_type": { - "type": "string", - "const": "sid", - "default": "sid", - "order": 0 - }, - "sid": { - "title": "System ID (SID)", - "type": "string", - "order": 1 - } - } - } - ] - }, - "username": { - "title": "User", - "description": "The username which is used to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "The password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true, - "order": 6 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 7 - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "The encryption method with is used when communicating with the database.", - "order": 8, - "oneOf": [ - { - "title": "Native Network Encryption (NNE)", - "description": "The native network encryption gives you the ability to encrypt database connections, without the configuration overhead of TCP/IP and SSL/TLS and without the need to open and listen on different ports.", - "required": ["encryption_method"], - "properties": { - "encryption_method": { - "type": "string", - "const": "client_nne", - "enum": ["client_nne"], - "default": "client_nne" - }, - "encryption_algorithm": { - "type": "string", - "description": "This parameter defines what encryption algorithm is used.", - "title": "Encryption Algorithm", - "default": "AES256", - "enum": ["AES256", "RC4_56", "3DES168"] - } - } - }, - { - "title": "TLS Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": ["encryption_method", "ssl_certificate"], - "properties": { - "encryption_method": { - "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" - }, - "ssl_certificate": { - "title": "SSL PEM File", - "description": "Privacy Enhanced Mail (PEM) files are concatenated certificate containers frequently used in certificate installations.", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "7f0455fb-4518-4ec0-b7a3-d808bf8081cc", - "name": "Orb", - "dockerRepository": "airbyte/source-orb", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/orb", - "icon": "orb.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.withorb.com/", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Orb Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "title": "Orb API Key", - "description": "Orb API Key, issued from the Orb admin console.", - "airbyte_secret": true, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2022-03-01T00:00:00Z. Any data with created_at before this data will not be synced.", - "examples": ["2022-03-01T00:00:00Z"], - "order": 2 - }, - "lookback_window_days": { - "type": "integer", - "title": "Lookback Window (in days)", - "default": 0, - "minimum": 0, - "description": "When set to N, the connector will always refresh resources created within the past N days. By default, updated objects that are not newly created are not incrementally synced.", - "order": 3 - }, - "string_event_properties_keys": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Event properties keys (string values)", - "description": "Property key names to extract from all events, in order to enrich ledger entries corresponding to an event deduction.", - "order": 4 - }, - "numeric_event_properties_keys": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Event properties keys (numeric values)", - "description": "Property key names to extract from all events, in order to enrich ledger entries corresponding to an event deduction.", - "order": 5 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a", - "name": "Orbit", - "dockerRepository": "airbyte/source-orbit", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/orbit", - "icon": "orbit.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/orbit", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Orbit Source Spec", - "type": "object", - "required": ["api_token", "workspace"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "airbyte_secret": true, - "title": "API Token", - "description": "Authorizes you to work with Orbit workspaces associated with the token.", - "order": 0 - }, - "workspace": { - "type": "string", - "title": "Workspace", - "description": "The unique name of the workspace that your API token is associated with.", - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "Date in the format 2022-06-26. Only load members whose last activities are after this date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 2 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239", - "name": "Paypal Transaction", - "dockerRepository": "airbyte/source-paypal-transaction", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transaction", - "icon": "paypal.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transactions", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Paypal Transaction Search", - "type": "object", - "required": ["start_date", "is_sandbox"], - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Paypal developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client secret", - "description": "The Client Secret of your Paypal developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh token (Optional)", - "description": "The key to refresh the expired access token.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "Start Date for data extraction in ISO format. Date must be in range from 3 years till 12 hrs before present time.", - "examples": ["2021-06-11T23:59:59-00:00"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" - }, - "is_sandbox": { - "title": "Sandbox", - "description": "Determines whether to use the sandbox or production environment.", - "type": "boolean", - "default": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "3052c77e-8b91-47e2-97a0-a29a22794b4b", - "name": "PersistIq", - "dockerRepository": "airbyte/source-persistiq", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/persistiq", - "icon": "persistiq.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/persistiq", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Persistiq Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "PersistIq API Key. See the docs for more information on where to find that key.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "6371b14b-bc68-4236-bfbd-468e8df8e968", - "name": "PokeAPI", - "dockerRepository": "airbyte/source-pokeapi", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/pokeapi", - "icon": "pokeapi.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/pokeapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pokeapi Spec", - "type": "object", - "required": ["pokemon_name"], - "additionalProperties": false, - "properties": { - "pokemon_name": { - "type": "string", - "title": "Pokemon Name", - "description": "Pokemon requested from the API.", - "pattern": "^[a-z0-9_\\-]+$", - "examples": ["ditto", "luxray", "snorlax"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "af6d50ee-dddf-4126-a8ee-7faee990774f", - "name": "PostHog", - "dockerRepository": "airbyte/source-posthog", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/posthog", - "icon": "posthog.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/posthog", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "PostHog Spec", - "type": "object", - "required": ["api_key", "start_date"], - "properties": { - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate the data. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "api_key": { - "type": "string", - "airbyte_secret": true, - "title": "API Key", - "description": "API Key. See the docs for information on how to generate this key." - }, - "base_url": { - "type": "string", - "default": "https://app.posthog.com", - "title": "Base URL", - "description": "Base PostHog url. Defaults to PostHog Cloud (https://app.posthog.com).", - "examples": ["https://posthog.example.com"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", - "name": "Postgres", - "dockerRepository": "airbyte/source-postgres-strict-encrypt", - "dockerImageTag": "1.0.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres", - "icon": "postgresql.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5432, - "examples": ["5432"], - "order": 1 - }, - "database": { - "title": "Database Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas (case sensitive) to sync from. Defaults to public.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "default": ["public"], - "order": 3 - }, - "username": { - "title": "Username", - "description": "Username to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", - "title": "JDBC URL Parameters (Advanced)", - "type": "string", - "order": 6 - }, - "ssl_mode": { - "title": "SSL Modes", - "description": "SSL connection modes. \n

  • disable - Disables encryption of communication between Airbyte and source database
    • \n
  • allow - Enables encryption only when required by the source database
    • \n
  • prefer - allows unencrypted connection only if the source database does not support encryption
    • \n
  • require - Always require encryption. If the source database server does not support encryption, connection will fail
    • \n
  • verify-ca - Always require encryption and verifies that the source database server has a valid SSL certificate
    • \n
  • verify-full - This is the most secure mode. Always require encryption and verifies the identity of the source database server
\n Read more in the docs.", - "type": "object", - "order": 7, - "oneOf": [ - { - "title": "allow", - "additionalProperties": false, - "description": "Allow SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "allow", - "enum": ["allow"], - "default": "allow", - "order": 0 - } - } - }, - { - "title": "prefer", - "additionalProperties": false, - "description": "Prefer SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "prefer", - "enum": ["prefer"], - "default": "prefer", - "order": 0 - } - } - }, - { - "title": "require", - "additionalProperties": false, - "description": "Require SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "require", - "enum": ["require"], - "default": "require", - "order": 0 - } - } - }, - { - "title": "verify-ca", - "additionalProperties": false, - "description": "Verify-ca SSL mode.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify-ca", - "enum": ["verify-ca"], - "default": "verify-ca", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_key_password": { - "type": "string", - "title": "Client Key Password (Optional)", - "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "verify-full", - "additionalProperties": false, - "description": "Verify-full SSL mode.", - "required": [ - "mode", - "ca_certificate", - "client_certificate", - "client_key" - ], - "properties": { - "mode": { - "type": "string", - "const": "verify-full", - "enum": ["verify-full"], - "default": "verify-full", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA Certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client Certificate", - "description": "Client certificate", - "airbyte_secret": true, - "multiline": true, - "order": 2 - }, - "client_key": { - "type": "string", - "title": "Client Key", - "description": "Client key", - "airbyte_secret": true, - "multiline": true, - "order": 3 - }, - "client_key_password": { - "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Replication Method", - "description": "Replication method for extracting data from the database.", - "order": 8, - "oneOf": [ - { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard", - "enum": ["Standard"], - "default": "Standard", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the docs.", - "required": ["method", "replication_slot", "publication"], - "properties": { - "method": { - "type": "string", - "const": "CDC", - "enum": ["CDC"], - "default": "CDC", - "order": 0 - }, - "plugin": { - "type": "string", - "title": "Plugin", - "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", - "enum": ["pgoutput", "wal2json"], - "default": "pgoutput", - "order": 1 - }, - "replication_slot": { - "type": "string", - "title": "Replication Slot", - "description": "A plugin logical replication slot. Read about replication slots.", - "order": 2 - }, - "publication": { - "type": "string", - "title": "Publication", - "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", - "order": 3 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "order": 4, - "min": 120, - "max": 1200 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "d60a46d4-709f-4092-a6b7-2457f7d455f5", - "name": "Prestashop", - "dockerRepository": "airbyte/source-prestashop", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/prestashop", - "icon": "prestashop.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "PrestaShop Spec", - "type": "object", - "required": ["url", "access_key"], - "additionalProperties": false, - "properties": { - "url": { - "type": "string", - "description": "Shop URL without trailing slash (domain name or IP address)" - }, - "access_key": { - "type": "string", - "description": "Your PrestaShop access key. See the docs for info on how to obtain this.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b08e4776-d1de-4e80-ab5c-1e51dad934a2", - "name": "Qualaroo", - "dockerRepository": "airbyte/source-qualaroo", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/qualaroo", - "icon": "qualaroo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/qualaroo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Qualaroo Spec", - "type": "object", - "required": ["token", "key", "start_date"], - "additionalProperties": true, - "properties": { - "token": { - "type": "string", - "title": "API token", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "key": { - "type": "string", - "title": "API key", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-03-01T00:00:00.000Z"] - }, - "survey_ids": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[0-9]{1,8}$" - }, - "title": "Qualaroo survey IDs", - "description": "IDs of the surveys from which you'd like to replicate data. If left empty, data from all surveys to which you have access will be replicated." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["token"], ["key"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "45d2e135-2ede-49e1-939f-3e3ec357a65e", - "name": "Recharge", - "dockerRepository": "airbyte/source-recharge", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/recharge", - "icon": "recharge.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/recharge", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Recharge Spec", - "type": "object", - "required": ["start_date", "access_token"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Recharge API, in the format YYYY-MM-DDT00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-05-14T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The value of the Access Token generated. See the docs for more information.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "cd42861b-01fc-4658-a8ab-5d11d0510f01", - "name": "Recurly", - "dockerRepository": "airbyte/source-recurly", - "dockerImageTag": "0.4.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/recurly", - "icon": "recurly.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/recurly", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Recurly Source Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "airbyte_secret": true, - "description": "Recurly API Key. See the docs for more information on how to generate this key.", - "order": 1 - }, - "begin_time": { - "type": "string", - "description": "ISO8601 timestamp from which the replication from Recurly API will start from.", - "examples": ["2021-12-01T00:00:00"], - "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$", - "order": 2 - }, - "end_time": { - "type": "string", - "description": "ISO8601 timestamp to which the replication from Recurly API will stop. Records after that date won't be imported.", - "examples": ["2021-12-01T00:00:00"], - "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$", - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6", - "name": "Redshift", - "dockerRepository": "airbyte/source-redshift", - "dockerImageTag": "0.3.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/redshift", - "icon": "redshift.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Redshift Source Spec", - "type": "object", - "required": ["host", "port", "database", "username", "password"], - "properties": { - "host": { - "title": "Host", - "description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com).", - "type": "string", - "order": 1 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5439, - "examples": ["5439"], - "order": 2 - }, - "database": { - "title": "Database", - "description": "Name of the database.", - "type": "string", - "examples": ["master"], - "order": 3 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas to sync from. Specify one or more explicitly or keep empty to process all schemas. Schema names are case sensitive.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "examples": ["public"], - "order": 4 - }, - "username": { - "title": "Username", - "description": "Username to use to access the database.", - "type": "string", - "order": 5 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 6 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "db04ecd1-42e7-4115-9cec-95812905c626", - "name": "Retently", - "dockerRepository": "airbyte/source-retently", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/retently", - "icon": "retently.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Retently Api Spec", - "type": "object", - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authentication Mechanism", - "description": "Choose how to authenticate to Retently", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "Authenticate via Retently (OAuth)", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "enum": ["Client"], - "default": "Client", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Retently developer application." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Retently developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "Retently Refresh Token which can be used to fetch new Bearer Tokens when the current one expires.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Authenticate with API Token", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Token", - "enum": ["Token"], - "default": "Token", - "order": 0 - }, - "api_key": { - "title": "API Token", - "description": "Retently API Token. See the docs for more information on how to obtain this key.", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "69589781-7828-43c5-9f63-8925b1c1ccc2", - "name": "S3", - "dockerRepository": "airbyte/source-s3", - "dockerImageTag": "0.1.18", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/s3", - "icon": "s3.svg", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/s3", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/s3", - "connectionSpecification": { - "title": "S3 Source Spec", - "type": "object", - "properties": { - "dataset": { - "title": "Output Stream Name", - "description": "The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", - "pattern": "^([A-Za-z0-9-_]+)$", - "order": 0, - "type": "string" - }, - "path_pattern": { - "title": "Pattern of files to replicate", - "description": "A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See this page to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern ** to pick up all files.", - "examples": [ - "**", - "myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv" - ], - "order": 10, - "type": "string" - }, - "format": { - "title": "File Format", - "description": "The format of the files you'd like to replicate", - "default": "csv", - "order": 20, - "type": "object", - "oneOf": [ - { - "title": "CSV", - "description": "This connector utilises PyArrow (Apache Arrow) for CSV parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "csv", - "type": "string" - }, - "delimiter": { - "title": "Delimiter", - "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", - "default": ",", - "minLength": 1, - "order": 0, - "type": "string" - }, - "infer_datatypes": { - "title": "Infer Datatypes", - "description": "Configures whether a schema for the source should be inferred from the current data or not. If set to false and a custom schema is set, then the manually enforced schema is used. If a schema is not manually set, and this is set to false, then all fields will be read as strings", - "default": true, - "order": 1, - "type": "boolean" - }, - "quote_char": { - "title": "Quote Character", - "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", - "default": "\"", - "order": 2, - "type": "string" - }, - "escape_char": { - "title": "Escape Character", - "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", - "order": 3, - "type": "string" - }, - "encoding": { - "title": "Encoding", - "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", - "default": "utf8", - "order": 4, - "type": "string" - }, - "double_quote": { - "title": "Double Quote", - "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", - "default": true, - "order": 5, - "type": "boolean" - }, - "newlines_in_values": { - "title": "Allow newlines in values", - "description": "Whether newline characters are allowed in CSV values. Turning this on may affect performance. Leave blank to default to False.", - "default": false, - "order": 6, - "type": "boolean" - }, - "additional_reader_options": { - "title": "Additional Reader Options", - "description": "Optionally add a valid JSON string here to provide additional options to the csv reader. Mappings must correspond to options detailed here. 'column_types' is used internally to handle schema so overriding that would likely cause problems.", - "default": "{}", - "examples": [ - "{\"timestamp_parsers\": [\"%m/%d/%Y %H:%M\", \"%Y/%m/%d %H:%M\"], \"strings_can_be_null\": true, \"null_values\": [\"NA\", \"NULL\"]}" - ], - "order": 7, - "type": "string" - }, - "advanced_options": { - "title": "Advanced Options", - "description": "Optionally add a valid JSON string here to provide additional Pyarrow ReadOptions. Specify 'column_names' here if your CSV doesn't have header, or if you want to use custom column names. 'block_size' and 'encoding' are already used above, specify them again here will override the values above.", - "default": "{}", - "examples": [ - "{\"column_names\": [\"column1\", \"column2\"]}" - ], - "order": 8, - "type": "string" - }, - "block_size": { - "title": "Block Size", - "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", - "default": 10000, - "order": 9, - "type": "integer" - } - } - }, - { - "title": "Parquet", - "description": "This connector utilises PyArrow (Apache Arrow) for Parquet parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "parquet", - "type": "string" - }, - "columns": { - "title": "Selected Columns", - "description": "If you only want to sync a subset of the columns from the file(s), add the columns you want here as a comma-delimited list. Leave it empty to sync all columns.", - "order": 0, - "type": "array", - "items": { - "type": "string" - } - }, - "batch_size": { - "title": "Record batch size", - "description": "Maximum number of records per batch read from the input files. Batches may be smaller if there aren’t enough rows in the file. This option can help avoid out-of-memory errors if your data is particularly wide.", - "default": 65536, - "order": 1, - "type": "integer" - }, - "buffer_size": { - "title": "Buffer Size", - "description": "Perform read buffering when deserializing individual column chunks. By default every group column will be loaded fully to memory. This option can help avoid out-of-memory errors if your data is particularly wide.", - "default": 2, - "type": "integer" - } - } - }, - { - "title": "Avro", - "description": "This connector utilises fastavro for Avro parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "avro", - "type": "string" - } - } - }, - { - "title": "Jsonl", - "description": "This connector uses PyArrow for JSON Lines (jsonl) file parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "jsonl", - "type": "string" - }, - "newlines_in_values": { - "title": "Allow newlines in values", - "description": "Whether newline characters are allowed in JSON values. Turning this on may affect performance. Leave blank to default to False.", - "default": false, - "order": 0, - "type": "boolean" - }, - "unexpected_field_behavior": { - "title": "Unexpected field behavior", - "description": "How JSON fields outside of explicit_schema (if given) are treated. Check PyArrow documentation for details", - "default": "infer", - "examples": ["ignore", "infer", "error"], - "order": 1, - "allOf": [ - { - "title": "UnexpectedFieldBehaviorEnum", - "description": "An enumeration.", - "enum": ["ignore", "infer", "error"], - "type": "string" - } - ] - }, - "block_size": { - "title": "Block Size", - "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", - "default": 10000, - "order": 2, - "type": "integer" - } - } - } - ] - }, - "schema": { - "title": "Manually enforced data schema (Optional)", - "description": "Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of { \"column\" : \"type\" }, where types are valid JSON Schema datatypes. Leave as {} to auto-infer the schema.", - "default": "{}", - "examples": [ - "{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}" - ], - "order": 30, - "type": "string" - }, - "provider": { - "title": "S3: Amazon Web Services", - "type": "object", - "properties": { - "bucket": { - "title": "Bucket", - "description": "Name of the S3 bucket where the file(s) exist.", - "order": 0, - "type": "string" - }, - "aws_access_key_id": { - "title": "AWS Access Key ID", - "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true, - "order": 1, - "type": "string" - }, - "aws_secret_access_key": { - "title": "AWS Secret Access Key", - "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true, - "order": 2, - "type": "string" - }, - "path_prefix": { - "title": "Path Prefix", - "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.", - "default": "", - "order": 3, - "type": "string" - }, - "endpoint": { - "title": "Endpoint", - "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.", - "default": "", - "order": 4, - "type": "string" - }, - "use_ssl": { - "title": "Use TLS", - "description": "Whether the remote server is using a secure SSL/TLS connection. Only relevant if using an S3-compatible, non-AWS server", - "order": 5, - "type": "boolean" - }, - "verify_ssl_cert": { - "title": "Verify TLS Certificates", - "description": "Set this to false to allow self signed certificates. Only relevant if using an S3-compatible, non-AWS server", - "order": 6, - "type": "boolean" - } - }, - "required": ["bucket"], - "order": 11, - "description": "Use this to load files from S3 or S3-compatible services" - } - }, - "required": ["dataset", "path_pattern", "provider"] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "a827c52e-791c-4135-a245-e233c5255199", - "name": "SFTP", - "dockerRepository": "airbyte/source-sftp", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/sftp", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/source/sftp", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "SFTP Source Spec", - "type": "object", - "required": ["user", "host", "port"], - "additionalProperties": true, - "properties": { - "user": { - "title": "User Name", - "description": "The server user", - "type": "string", - "order": 0 - }, - "host": { - "title": "Host Address", - "description": "The server host address", - "type": "string", - "examples": ["www.host.com", "192.0.2.1"], - "order": 1 - }, - "port": { - "title": "Port", - "description": "The server port", - "type": "integer", - "default": 22, - "examples": ["22"], - "order": 2 - }, - "credentials": { - "type": "object", - "title": "Authentication *", - "description": "The server authentication method", - "order": 3, - "oneOf": [ - { - "title": "Password Authentication", - "required": ["auth_method", "auth_user_password"], - "properties": { - "auth_method": { - "description": "Connect through password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "auth_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 1 - } - } - }, - { - "title": "SSH Key Authentication", - "required": ["auth_method", "auth_ssh_key"], - "properties": { - "auth_method": { - "description": "Connect through ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "auth_ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 1 - } - } - } - ] - }, - "file_types": { - "title": "File types", - "description": "Coma separated file types. Currently only 'csv' and 'json' types are supported.", - "type": "string", - "default": "csv,json", - "order": 4, - "examples": ["csv,json", "csv"] - }, - "folder_path": { - "title": "Folder Path (Optional)", - "description": "The directory to search files for sync", - "type": "string", - "default": "", - "examples": ["/logs/2022"], - "order": 5 - }, - "file_pattern": { - "title": "File Pattern (Optional)", - "description": "The regular expression to specify files for sync in a chosen Folder Path", - "type": "string", - "default": "", - "examples": [ - "log-([0-9]{4})([0-9]{2})([0-9]{2}) - This will filter files which `log-yearmmdd`" - ], - "order": 6 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b117307c-14b6-41aa-9422-947e34922962", - "name": "Salesforce", - "dockerRepository": "airbyte/source-salesforce", - "dockerImageTag": "1.0.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/salesforce", - "icon": "salesforce.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/salesforce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Salesforce Source Spec", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true, - "properties": { - "is_sandbox": { - "title": "Sandbox", - "description": "Toggle if you're using a Salesforce Sandbox", - "type": "boolean", - "default": false, - "order": 1 - }, - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "description": "Enter your Salesforce developer application's Client ID", - "type": "string", - "order": 2 - }, - "client_secret": { - "title": "Client Secret", - "description": "Enter your Salesforce developer application's Client secret", - "type": "string", - "airbyte_secret": true, - "order": 3 - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Enter your application's Salesforce Refresh Token used for Airbyte to access your Salesforce account.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "start_date": { - "title": "Start Date", - "description": "Enter the date in the YYYY-MM-DD format. Airbyte will replicate the data added on and after this date. If this field is blank, Airbyte will replicate all data.", - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2021-07-25", "2021-07-25T00:00:00Z"], - "order": 5 - }, - "streams_criteria": { - "type": "array", - "order": 6, - "items": { - "type": "object", - "required": ["criteria", "value"], - "properties": { - "criteria": { - "type": "string", - "title": "Search criteria", - "enum": [ - "starts with", - "ends with", - "contains", - "exacts", - "starts not with", - "ends not with", - "not contains", - "not exacts" - ], - "order": 1, - "default": "contains" - }, - "value": { - "type": "string", - "title": "Search value", - "order": 2 - } - } - }, - "title": "Filter Salesforce Objects (Optional)", - "description": "Filter streams relevant to you" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "is_sandbox": { - "type": "boolean", - "path_in_connector_config": ["is_sandbox"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "2470e835-feaf-4db6-96f3-70fd645acc77", - "name": "Salesforce (Singer)", - "dockerRepository": "airbyte/source-salesforce-singer", - "dockerImageTag": "0.2.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/salesforce", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/salesforce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Salesforce Source Spec", - "type": "object", - "_comment": "todo (cgardens) - allow default fields. is_sandbox and api_type should not be required and fall back on default. depends on change in api.", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "start_date", - "api_type" - ], - "additionalProperties": false, - "properties": { - "client_id": { - "description": "The Consumer Key that can be found when viewing your app in Salesforce", - "type": "string" - }, - "client_secret": { - "description": "The Consumer Secret that can be found when viewing your app in Salesforce", - "type": "string", - "airbyte_secret": true - }, - "refresh_token": { - "description": "Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow this guide to retrieve it.", - "type": "string", - "airbyte_secret": true - }, - "start_date": { - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"] - }, - "is_sandbox": { - "description": "Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. We provide more info on this field in the docs.", - "type": "boolean", - "default": false - }, - "api_type": { - "description": "Unless you know that you are transferring a very small amount of data, prefer using the BULK API. This will help avoid using up all of your API call quota with Salesforce. Valid values are BULK or REST.", - "type": "string", - "enum": ["BULK", "REST"], - "default": "BULK" - }, - "quota_percent_per_run": { - "description": "determines the maximum allowed API quota percentage the connector is allowed to consume per sync job", - "type": ["number", null], - "default": null - }, - "quota_percent_total": { - "description": "Determines the maximum allowed API quota percentage the connector is allowed to consume at any time", - "type": ["number", "null"], - "default": null - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87", - "name": "Sendgrid", - "dockerRepository": "airbyte/source-sendgrid", - "dockerImageTag": "0.2.8", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sendgrid", - "icon": "sendgrid.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sendgrid", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Sendgrid Spec", - "type": "object", - "required": ["apikey"], - "additionalProperties": false, - "properties": { - "apikey": { - "title": "Sendgrid API key", - "type": "string", - "description": "API Key, use admin to generate this key.", - "order": 0 - }, - "start_time": { - "title": "Start time", - "type": "integer", - "description": "Start time in timestamp integer format. Any data before this timestamp will not be replicated.", - "examples": [1558359837], - "order": 1 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "cdaf146a-9b75-49fd-9dd2-9d64a0bb4781", - "name": "Sentry", - "dockerRepository": "airbyte/source-sentry", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sentry", - "icon": "sentry.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sentry", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Sentry Spec", - "type": "object", - "required": ["auth_token", "organization", "project"], - "additionalProperties": false, - "properties": { - "auth_token": { - "type": "string", - "title": "Authentication Tokens", - "description": "Log into Sentry and then create authentication tokens.For self-hosted, you can find or create authentication tokens by visiting \"{instance_url_prefix}/settings/account/api/auth-tokens/\"", - "airbyte_secret": true - }, - "hostname": { - "type": "string", - "title": "Host Name", - "description": "Host name of Sentry API server.For self-hosted, specify your host name here. Otherwise, leave it empty.", - "default": "sentry.io" - }, - "organization": { - "type": "string", - "title": "Organization", - "description": "The slug of the organization the groups belong to." - }, - "project": { - "type": "string", - "title": "Project", - "description": "The name (slug) of the Project you want to sync." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", - "name": "Shopify", - "dockerRepository": "airbyte/source-shopify", - "dockerImageTag": "0.1.37", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify", - "icon": "shopify.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Shopify Source CDK Specifications", - "type": "object", - "required": ["shop", "start_date"], - "additionalProperties": true, - "properties": { - "shop": { - "type": "string", - "title": "Shopify Store", - "description": "The name of your Shopify store found in the URL. For example, if your URL was https://NAME.myshopify.com, then the name would be 'NAME'.", - "order": 1 - }, - "credentials": { - "title": "Shopify Authorization Method", - "description": "The authorization method to use to retrieve data from Shopify", - "type": "object", - "order": 2, - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "description": "OAuth2.0", - "required": ["auth_method"], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of the Shopify developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of the Shopify developer application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The Access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "title": "API Password", - "description": "API Password Auth", - "type": "object", - "required": ["auth_method", "api_password"], - "properties": { - "auth_method": { - "type": "string", - "const": "api_password", - "order": 0 - }, - "api_password": { - "type": "string", - "title": "API Password", - "description": "The API Password for your private application in the `Shopify` store.", - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "type": "string", - "title": "Replication Start Date", - "description": "The date you would like to replicate data from. Format: YYYY-MM-DD. Any data before this date will not be replicated.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "shop": { - "type": "string", - "path_in_connector_config": ["shop"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "2fed2292-5586-480c-af92-9944e39fe12d", - "name": "Short.io", - "dockerRepository": "airbyte/source-shortio", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/shortio", - "icon": "short.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://developers.short.io/reference", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Shortio Spec", - "type": "object", - "required": ["domain_id", "secret_key", "start_date"], - "properties": { - "domain_id": { - "type": "string", - "desciprtion": "Short.io Domain ID", - "title": "Domain ID", - "airbyte_secret": false - }, - "secret_key": { - "type": "string", - "title": "Secret Key", - "description": "Short.io Secret Key", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "airbyte_secret": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "374ebc65-6636-4ea0-925c-7d35999a8ffc", - "name": "Smartsheets", - "dockerRepository": "airbyte/source-smartsheets", - "dockerImageTag": "0.1.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/smartsheets", - "icon": "smartsheet.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/smartsheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Smartsheets Source Spec", - "type": "object", - "required": ["access_token", "spreadsheet_id"], - "additionalProperties": true, - "properties": { - "access_token": { - "title": "Access Token", - "description": "The access token to use for accessing your data from Smartsheets. This access token must be generated by a user with at least read access to the data you'd like to replicate. Generate an access token in the Smartsheets main menu by clicking Account > Apps & Integrations > API Access. See the setup guide for information on how to obtain this token.", - "type": "string", - "order": 0, - "airbyte_secret": true - }, - "spreadsheet_id": { - "title": "Sheet ID", - "description": "The spreadsheet ID. Find it by opening the spreadsheet then navigating to File > Properties", - "type": "string", - "order": 1 - }, - "start_datetime": { - "title": "Start Datetime (Optional)", - "type": "string", - "examples": ["2000-01-01T13:00:00", "2000-01-01T13:00:00-07:00"], - "description": "Only rows modified after this date/time will be replicated. This should be an ISO 8601 string, for instance: `2000-01-01T13:00:00`", - "format": "date-time", - "default": "2020-01-01T00:00:00+00:00", - "order": 2, - "airbyte_hidden": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": [], - "predicate_value": "", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": {} - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b", - "name": "Snapchat Marketing", - "dockerRepository": "airbyte/source-snapchat-marketing", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snapchat-marketing", - "icon": "snapchat.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snapchat-marketing", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Snapchat Marketing Spec", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Snapchat developer application.", - "airbyte_secret": true, - "order": 0 - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Snapchat developer application.", - "airbyte_secret": true, - "order": 1 - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "Refresh Token to renew the expired Access Token.", - "airbyte_secret": true, - "order": 2 - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "Date in the format 2022-01-01. Any data before this date will not be replicated.", - "examples": ["2022-01-01"], - "default": "2022-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 3 - }, - "end_date": { - "type": "string", - "title": "End Date (Optional)", - "description": "Date in the format 2017-01-25. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2022-01-30"], - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2", - "name": "Snowflake", - "dockerRepository": "airbyte/source-snowflake", - "dockerImageTag": "0.1.19", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake", - "icon": "snowflake.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Snowflake Source Spec", - "type": "object", - "required": ["host", "role", "warehouse", "database"], - "properties": { - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "order": 0, - "required": ["client_id", "client_secret", "auth_type"], - "properties": { - "auth_type": { - "type": "string", - "const": "OAuth", - "default": "OAuth", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Snowflake developer application.", - "airbyte_secret": true, - "order": 1 - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Snowflake developer application.", - "airbyte_secret": true, - "order": 2 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true, - "order": 3 - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token for making authenticated requests.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "Username and Password", - "type": "object", - "required": ["username", "password", "auth_type"], - "order": 1, - "properties": { - "auth_type": { - "type": "string", - "const": "username/password", - "default": "username/password", - "order": 0 - }, - "username": { - "description": "The username you created to allow Airbyte to access the database.", - "examples": ["AIRBYTE_USER"], - "type": "string", - "title": "Username", - "order": 1 - }, - "password": { - "description": "The password associated with the username.", - "type": "string", - "airbyte_secret": true, - "title": "Password", - "order": 2 - } - } - } - ], - "order": 0 - }, - "host": { - "description": "The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com).", - "examples": ["accountname.us-east-2.aws.snowflakecomputing.com"], - "type": "string", - "title": "Account Name", - "order": 1 - }, - "role": { - "description": "The role you created for Airbyte to access Snowflake.", - "examples": ["AIRBYTE_ROLE"], - "type": "string", - "title": "Role", - "order": 2 - }, - "warehouse": { - "description": "The warehouse you created for Airbyte to access data.", - "examples": ["AIRBYTE_WAREHOUSE"], - "type": "string", - "title": "Warehouse", - "order": 3 - }, - "database": { - "description": "The database you created for Airbyte to access data.", - "examples": ["AIRBYTE_DATABASE"], - "type": "string", - "title": "Database", - "order": 4 - }, - "schema": { - "description": "The source Snowflake schema tables. Leave empty to access tables from multiple schemas.", - "examples": ["AIRBYTE_SCHEMA"], - "type": "string", - "title": "Schema", - "order": 5 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 6 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "OAuth", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "host": { - "type": "string", - "path_in_connector_config": ["host"] - }, - "role": { - "type": "string", - "path_in_connector_config": ["role"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", - "name": "Square", - "dockerRepository": "airbyte/source-square", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/square", - "icon": "square.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/square", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Square Source CDK Specifications", - "type": "object", - "required": ["is_sandbox"], - "additionalProperties": true, - "properties": { - "is_sandbox": { - "type": "boolean", - "description": "Determines whether to use the sandbox or production environment.", - "title": "Sandbox", - "examples": [true, false], - "default": false - }, - "start_date": { - "type": "string", - "description": "UTC date in the format YYYY-MM-DD. Any data before this date will not be replicated. If not set, all data will be replicated.", - "title": "Start Date", - "examples": ["2021-01-01"], - "default": "2021-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "include_deleted_objects": { - "type": "boolean", - "description": "In some streams there is an option to include deleted objects (Items, Categories, Discounts, Taxes)", - "title": "Include Deleted Objects", - "examples": [true, false], - "default": false - }, - "credentials": { - "type": "object", - "title": "Credential Type", - "oneOf": [ - { - "title": "Oauth authentication", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Oauth", - "enum": ["Oauth"], - "default": "Oauth", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Square-issued ID of your application", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Square-issued application secret for your application", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "A refresh token generated using the above client ID and secret", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Key", - "required": ["auth_type", "api_key"], - "properties": { - "auth_type": { - "type": "string", - "const": "Apikey", - "enum": ["Apikey"], - "default": "Apikey", - "order": 1 - }, - "api_key": { - "title": "API key token", - "type": "string", - "description": "The API key for a Square application", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Oauth", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de", - "name": "Stripe", - "dockerRepository": "airbyte/source-stripe", - "dockerImageTag": "0.1.37", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", - "icon": "stripe.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", - "protocol_version": "0.2.1", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["client_secret", "account_id", "start_date"], - "properties": { - "account_id": { - "type": "string", - "title": "Account ID", - "description": "Your Stripe account ID (starts with 'acct_', find yours here).", - "order": 0 - }, - "client_secret": { - "type": "string", - "title": "Secret Key", - "description": "Stripe API key (usually starts with 'sk_live_'; find yours here).", - "airbyte_secret": true, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Replication start date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Only data generated after this date will be replicated.", - "examples": ["2017-01-25T00:00:00Z"], - "order": 2 - }, - "lookback_window_days": { - "type": "integer", - "title": "Lookback Window in days (Optional)", - "default": 0, - "minimum": 0, - "description": "When set, the connector will always re-export data from the past N days, where N is the value set here. This is useful if your data is frequently updated after creation. More info here", - "order": 3 - }, - "slice_range": { - "type": "integer", - "title": "Data request time increment in days (Optional)", - "default": 365, - "minimum": 1, - "examples": [1, 3, 10, 30, 180, 360], - "description": "The time increment used by the connector when requesting data from the Stripe API. The bigger the value is, the less requests will be made and faster the sync will be. On the other hand, the more seldom the state is persisted.", - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "badc5925-0485-42be-8caa-b34096cb71b5", - "name": "SurveyMonkey", - "dockerRepository": "airbyte/source-surveymonkey", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/surveymonkey", - "icon": "surveymonkey.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/surveymonkey", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "SurveyMonkey Spec", - "type": "object", - "required": ["access_token", "start_date"], - "additionalProperties": true, - "properties": { - "access_token": { - "title": "Access Token", - "order": 0, - "type": "string", - "airbyte_secret": true, - "description": "Access Token for making authenticated requests. See the docs for information on how to generate this key." - }, - "start_date": { - "title": "Start Date", - "order": 1, - "type": "string", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "survey_ids": { - "type": "array", - "order": 2, - "items": { - "type": "string", - "pattern": "^[0-9]{8,9}$" - }, - "title": "Survey Monkey survey IDs", - "description": "IDs of the surveys from which you'd like to replicate data. If left empty, data from all boards to which you have access will be replicated." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "d1aa448b-7c54-498e-ad95-263cbebcd2db", - "name": "Tempo", - "dockerRepository": "airbyte/source-tempo", - "dockerImageTag": "0.2.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tempo", - "icon": "tempo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Tempo Spec", - "type": "object", - "required": ["api_token"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "title": "API token", - "description": "Tempo API Token. Go to Tempo>Settings, scroll down to Data Access and select API integration.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35", - "name": "TikTok Marketing", - "dockerRepository": "airbyte/source-tiktok-marketing", - "dockerImageTag": "0.1.14", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "icon": "tiktok.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "connectionSpecification": { - "title": "TikTok Marketing Source Spec", - "type": "object", - "properties": { - "credentials": { - "title": "Authentication Method", - "description": "Authentication method", - "default": {}, - "order": 0, - "type": "object", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "app_id": { - "title": "App ID", - "description": "The Developer Application App ID.", - "airbyte_secret": true, - "type": "string" - }, - "secret": { - "title": "Secret", - "description": "The Developer Application Secret.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["app_id", "secret", "access_token"] - }, - { - "title": "Sandbox Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "sandbox_access_token", - "order": 0, - "type": "string" - }, - "advertiser_id": { - "title": "Advertiser ID", - "description": "The Advertiser ID which generated for the developer's Sandbox application.", - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The long-term authorized access token.", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["advertiser_id", "access_token"] - } - ] - }, - "start_date": { - "title": "Replication Start Date *", - "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", - "default": "2016-09-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 1, - "type": "string" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 2, - "type": "string" - }, - "report_granularity": { - "title": "Report Aggregation Granularity", - "description": "The granularity used for aggregating performance data in reports. See the docs.", - "enum": ["LIFETIME", "DAY", "HOUR"], - "order": 3, - "airbyte_hidden": true, - "type": "string" - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "title": "CompleteOauthOutputSpecification", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "path_in_connector_config": ["credentials", "access_token"], - "type": "string" - } - }, - "required": ["access_token"] - }, - "complete_oauth_server_input_specification": { - "title": "CompleteOauthServerInputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "type": "string" - }, - "secret": { - "title": "Secret", - "type": "string" - } - }, - "required": ["app_id", "secret"] - }, - "complete_oauth_server_output_specification": { - "title": "CompleteOauthServerOutputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "path_in_connector_config": ["credentials", "app_id"], - "type": "string" - }, - "secret": { - "title": "Secret", - "path_in_connector_config": ["credentials", "secret"], - "type": "string" - } - }, - "required": ["app_id", "secret"] - } - } - }, - "additionalProperties": true - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "b9dc6155-672e-42ea-b10d-9f1f1fb95ab1", - "name": "Twilio", - "dockerRepository": "airbyte/source-twilio", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/twilio", - "icon": "twilio.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/twilio", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Twilio Spec", - "type": "object", - "required": ["account_sid", "auth_token", "start_date"], - "additionalProperties": false, - "properties": { - "account_sid": { - "title": "Account ID", - "description": "Twilio account SID", - "airbyte_secret": true, - "type": "string", - "order": 1 - }, - "auth_token": { - "title": "Auth Token", - "description": "Twilio Auth Token.", - "airbyte_secret": true, - "type": "string", - "order": 2 - }, - "start_date": { - "title": "Replication Start Date", - "description": "UTC date and time in the format 2020-10-01T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2020-10-01T00:00:00Z"], - "type": "string", - "order": 3 - }, - "lookback_window": { - "title": "Lookback window", - "description": "How far into the past to look for records. (in minutes)", - "examples": [60], - "default": 0, - "type": "integer", - "order": 4 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e7eff203-90bf-43e5-a240-19ea3056c474", - "name": "Typeform", - "dockerRepository": "airbyte/source-typeform", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/typeform", - "icon": "typeform.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/typeform", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Typeform Spec", - "type": "object", - "required": ["token", "start_date"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "description": "The date you would like to replicate data. Format: YYYY-MM-DDTHH:mm:ss[Z].", - "examples": ["2020-01-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "token": { - "type": "string", - "description": "The API Token for a Typeform account.", - "airbyte_secret": true - }, - "form_ids": { - "title": "Form IDs to replicate", - "description": "When this parameter is set, the connector will replicate data only from the input forms. Otherwise, all forms in your Typeform account will be replicated. You can find form IDs in your form URLs. For example, in the URL \"https://mysite.typeform.com/to/u6nXL7\" the form_id is u6nXL7. You can find form URLs on Share panel", - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "c4cfaeda-c757-489a-8aba-859fb08b6970", - "name": "US Census", - "dockerRepository": "airbyte/source-us-census", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/us-census", - "icon": "uscensus.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/us-census", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "https://api.census.gov/ Source Spec", - "type": "object", - "required": ["api_key", "query_path"], - "additionalProperties": false, - "properties": { - "query_params": { - "type": "string", - "description": "The query parameters portion of the GET request, without the api key", - "pattern": "^\\w+=[\\w,:*]+(&(?!key)\\w+=[\\w,:*]+)*$", - "examples": [ - "get=NAME,NAICS2017_LABEL,LFO_LABEL,EMPSZES_LABEL,ESTAB,PAYANN,PAYQTR1,EMP&for=us:*&NAICS2017=72&LFO=001&EMPSZES=001", - "get=MOVEDIN,GEOID1,GEOID2,MOVEDOUT,FULL1_NAME,FULL2_NAME,MOVEDNET&for=county:*" - ] - }, - "query_path": { - "type": "string", - "description": "The path portion of the GET request", - "pattern": "^data(\\/[\\w\\d]+)+$", - "examples": [ - "data/2019/cbp", - "data/2018/acs", - "data/timeseries/healthins/sahie" - ] - }, - "api_key": { - "type": "string", - "description": "Your API Key. Get your key here.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "ef580275-d9a9-48bb-af5e-db0f5855be04", - "name": "Webflow", - "dockerRepository": "airbyte/source-webflow", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/webflow", - "icon": "webflow.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/webflow", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Webflow Spec", - "type": "object", - "required": ["api_key", "site_id"], - "additionalProperties": false, - "properties": { - "site_id": { - "title": "Site id", - "type": "string", - "description": "The id of the Webflow site you are requesting data from. See https://developers.webflow.com/#sites", - "example": "a relatively long hex sequence", - "order": 0 - }, - "api_key": { - "title": "API token", - "type": "string", - "description": "The API token for authenticating to Webflow. See https://university.webflow.com/lesson/intro-to-the-webflow-api", - "example": "a very long hex sequence", - "order": 1, - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "afa734e4-3571-11ec-991a-1e0031268139", - "name": "YouTube Analytics", - "dockerRepository": "airbyte/source-youtube-analytics", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/youtube-analytics", - "icon": "youtube.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/youtube-analytics", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "YouTube Analytics Spec", - "type": "object", - "required": ["credentials"], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authenticate via OAuth 2.0", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true, - "properties": { - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your developer application", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The client secret of your developer application", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "A refresh token generated using the above client ID and secret", - "airbyte_secret": true - } - } - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4", - "name": "Zendesk Chat", - "dockerRepository": "airbyte/source-zendesk-chat", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-chat", - "icon": "zendesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-chat", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zendesk Chat Spec", - "type": "object", - "required": ["start_date"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Zendesk Chat API, in the format YYYY-MM-DDT00:00:00Z.", - "examples": ["2021-02-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "subdomain": { - "type": "string", - "title": "Subdomain (Optional)", - "description": "Required if you access Zendesk Chat from a Zendesk Support subdomain.", - "default": "" - }, - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": ["credentials"], - "properties": { - "credentials": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to obtain new Access Token, when it's expired.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Access Token", - "required": ["credentials", "access_token"], - "properties": { - "credentials": { - "type": "string", - "const": "access_token", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The Access Token to make authenticated requests.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "credentials"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "325e0640-e7b3-4e24-b823-3361008f603f", - "name": "Zendesk Sunshine", - "dockerRepository": "airbyte/source-zendesk-sunshine", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-sunshine", - "icon": "zendesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk_sunshine", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zendesk Sunshine Spec", - "type": "object", - "required": ["start_date", "subdomain"], - "additionalProperties": true, - "properties": { - "subdomain": { - "title": "Subdomain", - "type": "string", - "description": "The subdomain for your Zendesk Account." - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_method", - "client_id", - "client_secret", - "access_token" - ], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "enum": ["oauth2.0"], - "default": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Long-term access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_method", "api_token", "email"], - "properties": { - "auth_method": { - "type": "string", - "const": "api_token", - "enum": ["api_token"], - "default": "api_token", - "order": 1 - }, - "api_token": { - "type": "string", - "title": "API Token", - "description": "API Token. See the docs for information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "title": "Email", - "description": "The user email for your Zendesk account" - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "79c1aa37-dae3-42ae-b333-d1c105477715", - "name": "Zendesk Support", - "dockerRepository": "airbyte/source-zendesk-support", - "dockerImageTag": "0.2.14", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-support", - "icon": "zendesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-support", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Zendesk Support Spec", - "type": "object", - "required": ["start_date", "subdomain"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Zendesk Support API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2020-10-15T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "subdomain": { - "type": "string", - "title": "Subdomain", - "description": "This is your Zendesk subdomain that can be found in your account URL. For example, in https://{MY_SUBDOMAIN}.zendesk.com/, where MY_SUBDOMAIN is the value of your subdomain." - }, - "credentials": { - "title": "Authentication *", - "type": "object", - "description": "Zendesk service provides two authentication methods. Choose between: `OAuth2.0` or `API token`.", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "required": ["access_token"], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The value of the API token generated. See the docs for more information.", - "airbyte_secret": true - } - } - }, - { - "title": "API Token", - "type": "object", - "required": ["email", "api_token"], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "string", - "const": "api_token", - "order": 0 - }, - "email": { - "title": "Email", - "type": "string", - "description": "The user email for your Zendesk account." - }, - "api_token": { - "title": "API Token", - "type": "string", - "description": "The value of the API token generated. See the docs for more information.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "credentials"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "f1e4c7f6-db5c-4035-981f-d35ab4998794", - "name": "Zenloop", - "dockerRepository": "airbyte/source-zenloop", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zenloop", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zenloop", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zenloop Spec", - "type": "object", - "required": ["api_token"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "description": "Zenloop API Token. You can get the API token in settings page here ", - "airbyte_secret": true - }, - "date_from": { - "type": "string", - "description": "Zenloop date_from. Format: 2021-10-24T03:30:30Z or 2021-10-24. Leave empty if only data from current data should be synced", - "examples": ["2021-10-24T03:30:30Z"] - }, - "survey_id": { - "type": "string", - "description": "Zenloop Survey ID. Can be found here. Leave empty to pull answers from all surveys", - "airbyte_secret": true - }, - "survey_group_id": { - "type": "string", - "description": "Zenloop Survey Group ID. Can be found by pulling All Survey Groups via SurveyGroups stream. Leave empty to pull answers from all survey groups", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5", - "name": "Zuora", - "dockerRepository": "airbyte/source-zuora", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zuora", - "icon": "zuora.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zuora", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zuora Connector Configuration", - "type": "object", - "required": ["start_date", "client_id", "client_secret"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "description": "Start Date in format: YYYY-MM-DD", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "window_in_days": { - "type": "integer", - "description": "The amount of days for each data-chunk begining from start_date. Bigger the value - faster the fetch. (Min=1, as for a Day; Max=364, as for a Year).", - "examples": [30, 60, 90, 120, 200, 364], - "default": 90 - }, - "client_id": { - "type": "string", - "description": "Client ID", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "description": "Client Secret", - "airbyte_secret": true - }, - "is_sandbox": { - "type": "boolean", - "description": "Defines whether use the SANDBOX or PRODUCTION environment.", - "default": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - } - ] -} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index cc3b7e90aaf5fe..60cca471e6a068 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -13,7 +13,6 @@ dependencies { compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java index 564e3d3ade85ec..3e9f7f05132cc2 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java @@ -103,6 +103,7 @@ public void flush(final StreamDescriptor decs, final Stream messages, final Ai protected static void endSync(final AirbyteDestination destination) throws Exception { destination.notifyEndOfInput(); + // Wait until process is finished cleanly. + while (!destination.isFinished()) { + Thread.sleep(1000); + } destination.close(); } diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 641822eeb211e9..0633a2c5a7f3b5 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.59.0 +current_version = 0.59.1 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index a02b1f11514272..bbcc00f7ebe8d7 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.59.1 +Fix concurrent CDK deadlock + ## 0.59.0 Fix state message handling when running concurrent syncs diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index f095029dbacec5..5a8626ad778164 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.59.0 +LABEL io.airbyte.version=0.59.1 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py index f37b78960a81f0..f7d65d31aca707 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py @@ -16,7 +16,6 @@ from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.record import Record -from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger @@ -26,7 +25,7 @@ class ConcurrentSource: A Source that reads data from multiple AbstractStreams concurrently. It does so by submitting partition generation, and partition read tasks to a thread pool. The tasks asynchronously add their output to a shared queue. - The read is done when all partitions for all streams were generated and read. + The read is done when all partitions for all streams w ere generated and read. """ DEFAULT_TIMEOUT_SECONDS = 900 @@ -40,6 +39,9 @@ def create( message_repository: MessageRepository, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, ) -> "ConcurrentSource": + is_single_threaded = initial_number_of_partitions_to_generate == 1 and num_workers == 1 + too_many_generator = not is_single_threaded and initial_number_of_partitions_to_generate >= num_workers + assert not too_many_generator, "It is required to have more workers than threads generating partitions" threadpool = ThreadPoolManager( concurrent.futures.ThreadPoolExecutor(max_workers=num_workers, thread_name_prefix="workerpool"), logger, @@ -83,10 +85,14 @@ def read( if not stream_instances_to_read_from: return - queue: ThrottledQueue = ThrottledQueue(Queue(), self._threadpool.get_throttler(), self._timeout_seconds) + # We set a maxsize to for the main thread to process record items when the queue size grows. This assumes that there are less + # threads generating partitions that than are max number of workers. If it weren't the case, we could have threads only generating + # partitions which would fill the queue. This number is arbitrarily set to 10_000 but will probably need to be changed given more + # information and might even need to be configurable depending on the source + queue: Queue[QueueItem] = Queue(maxsize=10_000) concurrent_stream_processor = ConcurrentReadProcessor( stream_instances_to_read_from, - PartitionEnqueuer(queue), + PartitionEnqueuer(queue, self._threadpool), self._threadpool, self._logger, self._slice_logger, @@ -113,10 +119,15 @@ def _submit_initial_partition_generators(self, concurrent_stream_processor: Conc def _consume_from_queue( self, - queue: ThrottledQueue, + queue: Queue[QueueItem], concurrent_stream_processor: ConcurrentReadProcessor, ) -> Iterable[AirbyteMessage]: while airbyte_message_or_record_or_exception := queue.get(): + try: + self._threadpool.shutdown_if_exception() + except Exception as exception: + concurrent_stream_processor.on_exception(exception) + yield from self._handle_item( airbyte_message_or_record_or_exception, concurrent_stream_processor, @@ -133,10 +144,8 @@ def _handle_item( # handle queue item and call the appropriate handler depending on the type of the queue item if isinstance(queue_item, Exception): yield from concurrent_stream_processor.on_exception(queue_item) - elif isinstance(queue_item, PartitionGenerationCompletedSentinel): yield from concurrent_stream_processor.on_partition_generation_completed(queue_item) - elif isinstance(queue_item, Partition): concurrent_stream_processor.on_partition(queue_item) elif isinstance(queue_item, PartitionCompleteSentinel): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py index 6c351850e62d2d..b6643042b24c45 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py @@ -1,6 +1,8 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Any + from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream @@ -15,3 +17,8 @@ def __init__(self, stream: AbstractStream): :param stream: The stream that was processed """ self.stream = stream + + def __eq__(self, other: Any) -> bool: + if isinstance(other, PartitionGenerationCompletedSentinel): + return self.stream == other.stream + return False diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py index 3c0eec206c54b3..04508b3ff1f119 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py @@ -2,10 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import logging +import threading from concurrent.futures import Future, ThreadPoolExecutor -from typing import Any, Callable, List - -from airbyte_cdk.sources.concurrent_source.throttler import Throttler +from typing import Any, Callable, List, Optional class ThreadPoolManager: @@ -13,34 +12,35 @@ class ThreadPoolManager: Wrapper to abstract away the threadpool and the logic to wait for pending tasks to be completed. """ - DEFAULT_SLEEP_TIME = 0.1 DEFAULT_MAX_QUEUE_SIZE = 10_000 def __init__( self, threadpool: ThreadPoolExecutor, logger: logging.Logger, - sleep_time: float = DEFAULT_SLEEP_TIME, max_concurrent_tasks: int = DEFAULT_MAX_QUEUE_SIZE, ): """ :param threadpool: The threadpool to use :param logger: The logger to use :param max_concurrent_tasks: The maximum number of tasks that can be pending at the same time - :param sleep_time: How long to sleep if there are too many pending tasks """ self._threadpool = threadpool self._logger = logger self._max_concurrent_tasks = max_concurrent_tasks self._futures: List[Future[Any]] = [] - self._throttler = Throttler(self._futures, sleep_time, max_concurrent_tasks) + self._lock = threading.Lock() + self._most_recently_seen_exception: Optional[Exception] = None - def get_throttler(self) -> Throttler: - return self._throttler + self._logging_threshold = max_concurrent_tasks * 2 - def submit(self, function: Callable[..., Any], *args: Any) -> None: - # Submit a task to the threadpool, removing completed tasks if there are too many tasks in self._futures. + def prune_to_validate_has_reached_futures_limit(self) -> bool: self._prune_futures(self._futures) + if len(self._futures) > self._logging_threshold: + self._logger.warning(f"ThreadPoolManager: The list of futures is getting bigger than expected ({len(self._futures)})") + return len(self._futures) >= self._max_concurrent_tasks + + def submit(self, function: Callable[..., Any], *args: Any) -> None: self._futures.append(self._threadpool.submit(function, *args)) def _prune_futures(self, futures: List[Future[Any]]) -> None: @@ -48,22 +48,24 @@ def _prune_futures(self, futures: List[Future[Any]]) -> None: Take a list in input and remove the futures that are completed. If a future has an exception, it'll raise and kill the stream operation. - Pruning this list safely relies on the assumptions that only the main thread can modify the list of futures. + We are using a lock here as without it, the algorithm would not be thread safe """ - if len(futures) < self._max_concurrent_tasks: - return + with self._lock: + if len(futures) < self._max_concurrent_tasks: + return - for index in reversed(range(len(futures))): - future = futures[index] + for index in reversed(range(len(futures))): + future = futures[index] - if future.done(): - # Only call future.exception() if the future is known to be done because it will block until the future is done. - # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Future.exception - optional_exception = future.exception() - if optional_exception: - exception = RuntimeError(f"Failed reading with error: {optional_exception}") - self._stop_and_raise_exception(exception) - futures.pop(index) + if future.done(): + # Only call future.exception() if the future is known to be done because it will block until the future is done. + # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Future.exception + optional_exception = future.exception() + if optional_exception: + # Exception handling should be done in the main thread. Hence, we only store the exception and expect the main + # thread to call raise_if_exception + self._most_recently_seen_exception = RuntimeError(f"Failed reading with error: {optional_exception}") + futures.pop(index) def shutdown(self) -> None: self._threadpool.shutdown(wait=False, cancel_futures=True) @@ -71,12 +73,21 @@ def shutdown(self) -> None: def is_done(self) -> bool: return all([f.done() for f in self._futures]) + def shutdown_if_exception(self) -> None: + """ + This method will raise if there is an exception so that the caller can use it. + """ + if self._most_recently_seen_exception: + self._stop_and_raise_exception(self._most_recently_seen_exception) + def check_for_errors_and_shutdown(self) -> None: """ Check if any of the futures have an exception, and raise it if so. If all futures are done, shutdown the threadpool. If the futures are not done, raise an exception. :return: """ + self.shutdown_if_exception() + exceptions_from_futures = [f for f in [future.exception() for future in self._futures] if f is not None] if exceptions_from_futures: exception = RuntimeError(f"Failed reading with errors: {exceptions_from_futures}") diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/throttler.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/throttler.py deleted file mode 100644 index 5b343caef1d78d..00000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/throttler.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import time -from concurrent.futures import Future -from typing import Any, List - - -class Throttler: - """ - A throttler that waits until the number of concurrent tasks is below a certain threshold. - """ - - def __init__(self, futures_list: List[Future[Any]], sleep_time: float, max_concurrent_tasks: int): - """ - :param futures_list: The list of futures to monitor - :param sleep_time: How long to sleep if there are too many pending tasks - :param max_concurrent_tasks: The maximum number of tasks that can be pending at the same time - """ - self._futures_list = futures_list - self._sleep_time = sleep_time - self._max_concurrent_tasks = max_concurrent_tasks - - def wait_and_acquire(self) -> None: - while len(self._futures_list) >= self._max_concurrent_tasks: - time.sleep(self._sleep_time) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py index 342c7ae3eec216..3869c6cf9e7326 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py @@ -1,10 +1,13 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import time +from queue import Queue from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream -from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue +from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem class PartitionEnqueuer: @@ -12,26 +15,40 @@ class PartitionEnqueuer: Generates partitions from a partition generator and puts them in a queue. """ - def __init__(self, queue: ThrottledQueue) -> None: + def __init__(self, queue: Queue[QueueItem], thread_pool_manager: ThreadPoolManager, sleep_time_in_seconds: float = 0.1) -> None: """ :param queue: The queue to put the partitions in. :param throttler: The throttler to use to throttle the partition generation. """ self._queue = queue + self._thread_pool_manager = thread_pool_manager + self._sleep_time_in_seconds = sleep_time_in_seconds def generate_partitions(self, stream: AbstractStream) -> None: """ Generate partitions from a partition generator and put them in a queue. When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated. - If an exception is encountered, the exception will be caught and put in the queue. + If an exception is encountered, the exception will be caught and put in the queue. This is very important because if we don't, the + main thread will have no way to know that something when wrong and will wait until the timeout is reached This method is meant to be called in a separate thread. - :param partition_generator: The partition Generator - :return: """ try: for partition in stream.generate_partitions(): + # Adding partitions to the queue generates futures. To avoid having too many futures, we throttle here. We understand that + # we might add more futures than the limit by throttling in the threads while it is the main thread that actual adds the + # future but we expect the delta between the max futures length and the actual to be small enough that it would not be an + # issue. We do this in the threads because we want the main thread to always be processing QueueItems as if it does not, the + # queue size could grow and generating OOM issues. + # + # Also note that we do not expect this to create deadlocks where all worker threads wait because we have less + # PartitionEnqueuer threads than worker threads. + # + # Also note that prune_to_validate_has_reached_futures_limit has a lock while pruning which might create a bottleneck in + # terms of performance. + while self._thread_pool_manager.prune_to_validate_has_reached_futures_limit(): + time.sleep(self._sleep_time_in_seconds) self._queue.put(partition) self._queue.put(PartitionGenerationCompletedSentinel(stream)) except Exception as e: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py index 28f920326cf43a..3df19ca29f9262 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py @@ -1,10 +1,10 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from queue import Queue from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition -from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue -from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem class PartitionReader: @@ -12,7 +12,7 @@ class PartitionReader: Generates records from a partition and puts them in a queue. """ - def __init__(self, queue: ThrottledQueue) -> None: + def __init__(self, queue: Queue[QueueItem]) -> None: """ :param queue: The queue to put the records in. """ @@ -23,7 +23,8 @@ def process_partition(self, partition: Partition) -> None: Process a partition and put the records in the output queue. When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated. - If an exception is encountered, the exception will be caught and put in the queue. + If an exception is encountered, the exception will be caught and put in the queue. This is very important because if we don't, the + main thread will have no way to know that something when wrong and will wait until the timeout is reached This method is meant to be called from a thread. :param partition: The partition to read data from diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py deleted file mode 100644 index 27a1757e47f5ad..00000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from queue import Queue - -from airbyte_cdk.sources.concurrent_source.throttler import Throttler -from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem - - -class ThrottledQueue: - """ - A queue that throttles the number of items that can be added to it. - - We throttle the queue using custom logic instead of relying on the queue's max size - because the main thread can continuously dequeue before submitting a future. - - Since the main thread doesn't wait, it'll be able to remove items from the queue even if the tasks should be throttled, - so the tasks won't wait. - - This class solves this issue by checking if we should throttle the queue before adding an item to it. - An example implementation of a throttler would check if the number of pending futures is greater than a certain threshold. - """ - - def __init__(self, queue: Queue[QueueItem], throttler: Throttler, timeout: float) -> None: - """ - :param queue: The queue to throttle - :param throttler: The throttler to use to throttle the queue - :param timeout: The timeout to use when getting items from the queue - """ - self._queue = queue - self._throttler = throttler - self._timeout = timeout - - def put(self, item: QueueItem) -> None: - self._throttler.wait_and_acquire() - self._queue.put(item) - - def get(self) -> QueueItem: - return self._queue.get(block=True, timeout=self._timeout) - - def empty(self) -> bool: - return self._queue.empty() diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 483305c3446933..1dac92aece31d9 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.59.0", + version="0.59.1", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py deleted file mode 100644 index d41b92fb8afa6b..00000000000000 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import Mock, call - -import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel -from airbyte_cdk.sources.streams.concurrent.adapters import StreamPartition -from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer -from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue - - -@pytest.mark.parametrize( - "slices", [pytest.param([], id="test_no_partitions"), pytest.param([{"partition": 1}, {"partition": 2}], id="test_two_partitions")] -) -def test_partition_generator(slices): - queue = Mock(spec=ThrottledQueue) - partition_generator = PartitionEnqueuer(queue) - - stream = Mock() - message_repository = Mock() - sync_mode = SyncMode.full_refresh - cursor_field = None - state = None - cursor = Mock() - partitions = [StreamPartition(stream, s, message_repository, sync_mode, cursor_field, state, cursor) for s in slices] - stream.generate_partitions.return_value = iter(partitions) - - partition_generator.generate_partitions(stream) - - assert queue.put.has_calls([call(p) for p in partitions] + [call(PartitionGenerationCompletedSentinel(stream))]) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py new file mode 100644 index 00000000000000..c3ce277fb8c70d --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import unittest +from queue import Queue +from typing import Callable, Iterable, List +from unittest.mock import Mock, patch + +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem + +_SOME_PARTITIONS: List[Partition] = [Mock(spec=Partition), Mock(spec=Partition)] + + +class PartitionEnqueuerTest(unittest.TestCase): + def setUp(self) -> None: + self._queue: Queue[QueueItem] = Queue() + self._thread_pool_manager = Mock(spec=ThreadPoolManager) + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = False + self._partition_generator = PartitionEnqueuer(self._queue, self._thread_pool_manager) + + @patch("airbyte_cdk.sources.streams.concurrent.partition_enqueuer.time.sleep") + def test_given_no_partitions_when_generate_partitions_then_do_not_wait(self, mocked_sleep): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = True # shouldn't be called but just in case + stream = self._a_stream([]) + + self._partition_generator.generate_partitions(stream) + + assert mocked_sleep.call_count == 0 + + def test_given_partitions_when_generate_partitions_then_only_push_sentinel(self): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = True + stream = self._a_stream([]) + + self._partition_generator.generate_partitions(stream) + + assert self._consume_queue() == [PartitionGenerationCompletedSentinel(stream)] + + def test_given_partitions_when_generate_partitions_then_return_partitions_before_sentinel(self): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = False + stream = self._a_stream(_SOME_PARTITIONS) + + self._partition_generator.generate_partitions(stream) + + assert self._consume_queue() == _SOME_PARTITIONS + [PartitionGenerationCompletedSentinel(stream)] + + @patch("airbyte_cdk.sources.streams.concurrent.partition_enqueuer.time.sleep") + def test_given_partition_but_limit_reached_when_generate_partitions_then_wait_until_not_hitting_limit(self, mocked_sleep): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.side_effect = [True, True, False] + stream = self._a_stream([Mock(spec=Partition)]) + + self._partition_generator.generate_partitions(stream) + + assert mocked_sleep.call_count == 2 + + def test_given_exception_when_generate_partitions_then_raise(self): + stream = Mock(spec=AbstractStream) + exception = ValueError() + stream.generate_partitions.side_effect = self._partitions_before_raising(_SOME_PARTITIONS, exception) + + self._partition_generator.generate_partitions(stream) + + assert self._consume_queue() == _SOME_PARTITIONS + [exception] + + def _partitions_before_raising(self, partitions: List[Partition], exception: Exception) -> Callable[[], Iterable[Partition]]: + def inner_function() -> Iterable[Partition]: + for partition in partitions: + yield partition + raise exception + return inner_function + + @staticmethod + def _a_stream(partitions: List[Partition]) -> AbstractStream: + stream = Mock(spec=AbstractStream) + stream.generate_partitions.return_value = iter(partitions) + return stream + + def _consume_queue(self) -> List[QueueItem]: + queue_content: List[QueueItem] = [] + while queue_item := self._queue.get(): + if isinstance(queue_item, (PartitionGenerationCompletedSentinel, Exception)): + queue_content.append(queue_item) + break + queue_content.append(queue_item) + return queue_content diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py index df82432c415f15..4b0742f991a139 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py @@ -1,32 +1,66 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import unittest from queue import Queue +from typing import Callable, Iterable, List from unittest.mock import Mock +import pytest from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.record import Record -from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem +_RECORDS = [ + Record({"id": 1, "name": "Jack"}, "stream"), + Record({"id": 2, "name": "John"}, "stream"), +] -def test_partition_reader(): - queue = Queue() - partition_reader = PartitionReader(queue) - stream_partition = Mock() - records = [ - Record({"id": 1, "name": "Jack"}, "stream"), - Record({"id": 2, "name": "John"}, "stream"), - ] - stream_partition.read.return_value = iter(records) +class PartitionReaderTest(unittest.TestCase): + def setUp(self) -> None: + self._queue: Queue[QueueItem] = Queue() + self._partition_reader = PartitionReader(self._queue) - partition_reader.process_partition(stream_partition) + def test_given_no_records_when_process_partition_then_only_emit_sentinel(self): + self._partition_reader.process_partition(self._a_partition([])) - actual_records = [] - while record := queue.get(): - if isinstance(record, PartitionCompleteSentinel): + while queue_item := self._queue.get(): + if not isinstance(queue_item, PartitionCompleteSentinel): + pytest.fail("Only one PartitionCompleteSentinel is expected") break - actual_records.append(record) - assert records == actual_records + def test_given_read_partition_successful_when_process_partition_then_queue_records_and_sentinel(self): + self._partition_reader.process_partition(self._a_partition(_RECORDS)) + + actual_records = [] + while queue_item := self._queue.get(): + if isinstance(queue_item, PartitionCompleteSentinel): + break + actual_records.append(queue_item) + + assert _RECORDS == actual_records + + def test_given_exception_when_process_partition_then_queue_records_and_raise_exception(self): + partition = Mock() + exception = ValueError() + partition.read.side_effect = self._read_with_exception(_RECORDS, exception) + + self._partition_reader.process_partition(partition) + + for i in range(len(_RECORDS)): + assert self._queue.get() == _RECORDS[i] + assert self._queue.get() == exception + + def _a_partition(self, records: List[Record]) -> Partition: + partition = Mock(spec=Partition) + partition.read.return_value = iter(records) + return partition + + @staticmethod + def _read_with_exception(records: List[Record], exception: Exception) -> Callable[[], Iterable[Record]]: + def mocked_function() -> Iterable[Record]: + yield from records + raise exception + return mocked_function diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py index db950f33f79ce9..102cf7cdd4482c 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py @@ -7,13 +7,11 @@ from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager -_SLEEP_TIME = 2 - class ThreadPoolManagerTest(TestCase): def setUp(self): self._threadpool = Mock(spec=ThreadPoolExecutor) - self._thread_pool_manager = ThreadPoolManager(self._threadpool, Mock(), max_concurrent_tasks=1, sleep_time=_SLEEP_TIME) + self._thread_pool_manager = ThreadPoolManager(self._threadpool, Mock(), max_concurrent_tasks=1) self._fn = lambda x: x self._arg = "arg" @@ -23,15 +21,38 @@ def test_submit_calls_underlying_thread_pool(self): assert len(self._thread_pool_manager._futures) == 1 - def test_submit_task_previous_task_failed(self): + def test_given_no_exceptions_when_shutdown_if_exception_then_do_not_raise(self): + future = Mock(spec=Future) + future.exception.return_value = None + future.done.side_effect = [True, True] + + self._thread_pool_manager._futures = [future] + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit() + + self._thread_pool_manager.shutdown_if_exception() # do not raise + + def test_given_exception_when_shutdown_if_exception_then_raise(self): + future = Mock(spec=Future) + future.exception.return_value = RuntimeError + future.done.side_effect = [True, True] + + self._thread_pool_manager._futures = [future] + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit() + + with self.assertRaises(RuntimeError): + self._thread_pool_manager.shutdown_if_exception() + + def test_given_exception_during_pruning_when_check_for_errors_and_shutdown_then_shutdown_and_raise(self): future = Mock(spec=Future) future.exception.return_value = RuntimeError future.done.side_effect = [True, True] self._thread_pool_manager._futures = [future] + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit() with self.assertRaises(RuntimeError): - self._thread_pool_manager.submit(self._fn, self._arg) + self._thread_pool_manager.check_for_errors_and_shutdown() + self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True) def test_shutdown(self): self._thread_pool_manager.shutdown() diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttled_queue.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttled_queue.py deleted file mode 100644 index 33e4c9f0a0587f..00000000000000 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttled_queue.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from queue import Queue -from unittest.mock import Mock - -import pytest -from _queue import Empty -from airbyte_cdk.sources.concurrent_source.throttler import Throttler -from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue - -_AN_ITEM = Mock() - - -def test_new_throttled_queue_is_empty(): - queue = Queue() - throttler = Mock(spec=Throttler) - timeout = 100 - throttled_queue = ThrottledQueue(queue, throttler, timeout) - - assert throttled_queue.empty() - - -def test_throttled_queue_is_not_empty_after_putting_an_item(): - queue = Queue() - throttler = Mock(spec=Throttler) - timeout = 100 - throttled_queue = ThrottledQueue(queue, throttler, timeout) - - throttled_queue.put(_AN_ITEM) - - assert not throttled_queue.empty() - - -def test_throttled_queue_get_returns_item_if_any(): - queue = Queue() - throttler = Mock(spec=Throttler) - timeout = 100 - throttled_queue = ThrottledQueue(queue, throttler, timeout) - - throttled_queue.put(_AN_ITEM) - item = throttled_queue.get() - - assert item == _AN_ITEM - assert throttled_queue.empty() - - -def test_throttled_queue_blocks_for_timeout_seconds_if_no_items(): - queue = Mock(spec=Queue) - throttler = Mock(spec=Throttler) - timeout = 100 - throttled_queue = ThrottledQueue(queue, throttler, timeout) - - throttled_queue.get() - - assert queue.get.is_called_once_with(block=True, timeout=timeout) - - -def test_throttled_queue_raises_an_error_if_no_items_after_timeout(): - queue = Queue() - throttler = Mock(spec=Throttler) - timeout = 0.001 - throttled_queue = ThrottledQueue(queue, throttler, timeout) - - with pytest.raises(Empty): - throttled_queue.get() diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttler.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttler.py deleted file mode 100644 index fbe006771244af..00000000000000 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttler.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from unittest.mock import patch - -from airbyte_cdk.sources.concurrent_source.throttler import Throttler - - -@patch('time.sleep', side_effect=lambda _: None) -@patch('airbyte_cdk.sources.concurrent_source.throttler.len', side_effect=[1, 1, 0]) -def test_throttler(sleep_mock, len_mock): - throttler = Throttler([], 0.1, 1) - throttler.wait_and_acquire() - assert sleep_mock.call_count == 3 diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index e762b5705d2484..771e7d03cdcfa6 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -491,6 +491,37 @@ This command runs formatting checks and reformats any code that would be reforma Running `airbyte-ci format fix all` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. +### `poetry` command subgroup + +Available commands: + +- `airbyte-ci poetry publish` + +### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ---------------- | -------- | ------- | --------------------------- | -------------------------------------------------------------- | +| `--package-path` | True | | | The path to the python package to execute a poetry command on. | + +### Examples + +- Publish a python package: `airbyte-ci poetry --package-path=path/to/package publish --publish-name=my-package --publish-version="1.2.3" --python-registry-token="..." --registry-url="http://host.docker.internal:8012/"` + +### `publish` command + +This command publishes poetry packages (using `pyproject.toml`) or python packages (using `setup.py`) to a python registry. + +For poetry packages, the package name and version can be taken from the `pyproject.toml` file or be specified as options. + +#### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------- | -------- | ----------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------- | +| `--publish-name` | False | | | The name of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--publish-version` | False | | | The version of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--python-registry-token` | True | | PYTHON_REGISTRY_TOKEN | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | +| `--registry-url` | False | https://pypi.org/simple | | The python registry to publish to. Defaults to main pypi | + ### `metadata` command subgroup Available commands: @@ -547,7 +578,12 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | -| 3.5.3 | [#34339](https://github.com/airbytehq/airbyte/pull/34339) | only do minimal changes on a connector version_bump | +| 3.7.2 | [#34555](https://github.com/airbytehq/airbyte/pull/34555) | Override secret masking in some very specific special cases. | +| 3.7.1 | [#34441](https://github.com/airbytehq/airbyte/pull/34441) | Support masked secret scrubbing for java CDK v0.15+ | +| 3.7.0 | [#34343](https://github.com/airbytehq/airbyte/pull/34343) | allow running connector upgrade_cdk for java connectors | +| 3.6.1 | [#34490](https://github.com/airbytehq/airbyte/pull/34490) | Fix inconsistent dagger log path typing | +| 3.6.0 | [#34111](https://github.com/airbytehq/airbyte/pull/34111) | Add python registry publishing | +| 3.5.3 | [#34339](https://github.com/airbytehq/airbyte/pull/34339) | only do minimal changes on a connector version_bump | | 3.5.2 | [#34381](https://github.com/airbytehq/airbyte/pull/34381) | Bind a sidecar docker host for `airbyte-ci test` | | 3.5.1 | [#34321](https://github.com/airbytehq/airbyte/pull/34321) | Upgrade to Dagger 0.9.6 . | | 3.5.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Pass extra params after Gradle tasks. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index a4471bac7ecaf1..829ab07b4e0a03 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -89,12 +89,16 @@ def metadata_service_gcs_credentials_secret(self) -> Secret: def spec_cache_gcs_credentials_secret(self) -> Secret: return self.dagger_client.set_secret("spec_cache_gcs_credentials", self.spec_cache_gcs_credentials) + @property + def pre_release_suffix(self) -> str: + return self.git_revision[:10] + @property def docker_image_tag(self) -> str: # get the docker image tag from the parent class metadata_tag = super().docker_image_tag if self.pre_release: - return f"{metadata_tag}-dev.{self.git_revision[:10]}" + return f"{metadata_tag}-dev.{self.pre_release_suffix}" else: return metadata_tag diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index ba61a521ec66c3..c6e4002829c336 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -14,8 +14,10 @@ from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation +from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry, PythonRegistryPublishContext from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker +from pipelines.helpers.pip import is_package_published from pipelines.models.steps import Step, StepResult, StepStatus from pydantic import ValidationError @@ -52,6 +54,28 @@ async def _run(self) -> StepResult: return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") +class CheckPythonRegistryPackageDoesNotExist(Step): + context: PythonRegistryPublishContext + title = "Check if the connector is published on python registry" + + async def _run(self) -> StepResult: + is_published = is_package_published( + self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry + ) + if is_published: + return StepResult( + self, + status=StepStatus.SKIPPED, + stderr=f"{self.context.package_metadata.name} already exists in version {self.context.package_metadata.version}.", + ) + else: + return StepResult( + self, + status=StepStatus.SUCCESS, + stdout=f"{self.context.package_metadata.name} does not exist in version {self.context.package_metadata.version}.", + ) + + class PushConnectorImageToRegistry(Step): context: PublishConnectorContext title = "Push connector image to registry" @@ -259,6 +283,11 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run() results.append(check_connector_image_results) + python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context) + results.extend(python_registry_steps) + if terminate_early: + return create_connector_report(results) + # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file. # We also need to upload the spec to the spec cache bucket. if check_connector_image_results.status is StepStatus.SKIPPED: @@ -312,6 +341,33 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: return connector_report +async def _run_python_registry_publish_pipeline(context: PublishConnectorContext) -> Tuple[List[StepResult], bool]: + """ + Run the python registry publish pipeline for a single connector. + Return the results of the steps and a boolean indicating whether there was an error and the pipeline should be stopped. + """ + results: List[StepResult] = [] + # Try to convert the context to a PythonRegistryPublishContext. If it returns None, it means we don't need to publish to a python registry. + python_registry_context = await PythonRegistryPublishContext.from_publish_connector_context(context) + if not python_registry_context: + return results, False + + check_python_registry_package_exists_results = await CheckPythonRegistryPackageDoesNotExist(python_registry_context).run() + results.append(check_python_registry_package_exists_results) + if check_python_registry_package_exists_results.status is StepStatus.SKIPPED: + context.logger.info("The connector version is already published on python registry.") + elif check_python_registry_package_exists_results.status is StepStatus.SUCCESS: + context.logger.info("The connector version is not published on python registry. Let's build and publish it.") + publish_to_python_registry_results = await PublishToPythonRegistry(python_registry_context).run() + results.append(publish_to_python_registry_results) + if publish_to_python_registry_results.status is StepStatus.FAILURE: + return results, True + elif check_python_registry_package_exists_results.status is StepStatus.FAILURE: + return results, True + + return results, False + + def reorder_contexts(contexts: List[PublishConnectorContext]) -> List[PublishConnectorContext]: """Reorder contexts so that the ones that are for strict-encrypt/secure connectors come first. The metadata upload on publish checks if the the connectors referenced in the metadata file are already published to DockerHub. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py index ad1fd2e0e80a97..68c031cac62e97 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py @@ -3,26 +3,14 @@ # import asyncclick as click -import requests # type: ignore from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.upgrade_cdk.pipeline import run_connector_cdk_upgrade_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -def latest_cdk_version() -> str: - """ - Get the latest version of airbyte-cdk from pypi - """ - cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" - response = requests.get(cdk_pypi_url) - response.raise_for_status() - package_info = response.json() - return package_info["info"]["version"] - - @click.command(cls=DaggerPipelineCommand, short_help="Upgrade CDK version") -@click.argument("target-cdk-version", type=str, default=latest_cdk_version) +@click.argument("target-cdk-version", type=str, default="latest") @click.pass_context async def bump_version( ctx: click.Context, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py index 50390ca281ed8f..95839877f6c246 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py @@ -7,12 +7,17 @@ import re from typing import TYPE_CHECKING +from connector_ops.utils import ConnectorLanguage # type: ignore +from dagger import Directory from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report from pipelines.helpers import git +from pipelines.helpers.connectors import cdk_helpers from pipelines.models.steps import Step, StepResult, StepStatus if TYPE_CHECKING: + from typing import Optional + from anyio import Semaphore @@ -30,14 +35,22 @@ def __init__( async def _run(self) -> StepResult: context = self.context - og_connector_dir = await context.get_connector_dir() - if "setup.py" not in await og_connector_dir.entries(): - return self.skip("Connector does not have a setup.py file.") - setup_py = og_connector_dir.file("setup.py") - setup_py_content = await setup_py.contents() + try: - updated_setup_py = self.update_cdk_version(setup_py_content) - updated_connector_dir = og_connector_dir.with_new_file("setup.py", updated_setup_py) + og_connector_dir = await context.get_connector_dir() + if self.context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]: + updated_connector_dir = await self.upgrade_cdk_version_for_python_connector(og_connector_dir) + elif self.context.connector.language is ConnectorLanguage.JAVA: + updated_connector_dir = await self.upgrade_cdk_version_for_java_connector(og_connector_dir) + else: + return StepResult( + self, + StepStatus.FAILURE, + stderr=f"No CDK for connector {self.context.connector.technical_name} of written in {self.context.connector.language}", + ) + + if updated_connector_dir is None: + return self.skip(self.skip_reason) diff = og_connector_dir.diff(updated_connector_dir) exported_successfully = await diff.export(os.path.join(git.get_git_repo_path(), context.connector.code_directory)) if not exported_successfully: @@ -55,17 +68,57 @@ async def _run(self) -> StepResult: exc_info=e, ) - def update_cdk_version(self, og_setup_py_content: str) -> str: + async def upgrade_cdk_version_for_java_connector(self, og_connector_dir: Directory) -> Directory: + if "build.gradle" not in await og_connector_dir.entries(): + raise ValueError(f"Java connector {self.context.connector.technical_name} does not have a build.gradle file.") + + build_gradle = og_connector_dir.file("build.gradle") + build_gradle_content = await build_gradle.contents() + + old_cdk_version_required = re.search(r"cdkVersionRequired *= *'(?P[0-9]*\.[0-9]*\.[0-9]*)?'", build_gradle_content) + # If there is no airbyte-cdk dependency, add the version + if old_cdk_version_required is None: + raise ValueError("Could not find airbyte-cdk dependency in build.gradle") + + if self.new_version == "latest": + new_version = await cdk_helpers.get_latest_java_cdk_version(self.context.get_repo_dir()) + else: + new_version = self.new_version + + updated_build_gradle = build_gradle_content.replace(old_cdk_version_required.group("version"), new_version) + + use_local_cdk = re.search(r"useLocalCdk *=.*", updated_build_gradle) + if use_local_cdk is not None: + updated_build_gradle = updated_build_gradle.replace(use_local_cdk.group(), "useLocalCdk = false") + + return og_connector_dir.with_new_file("build.gradle", updated_build_gradle) + + async def upgrade_cdk_version_for_python_connector(self, og_connector_dir: Directory) -> Optional[Directory]: + context = self.context + og_connector_dir = await context.get_connector_dir() + if "setup.py" not in await og_connector_dir.entries(): + self.skip_reason = f"Python connector {self.context.connector.technical_name} does not have a setup.py file." + return None + setup_py = og_connector_dir.file("setup.py") + setup_py_content = await setup_py.contents() + airbyte_cdk_dependency = re.search( - r"airbyte-cdk(?P\[[a-zA-Z0-9-]*\])?(?P[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?", og_setup_py_content + r"airbyte-cdk(?P\[[a-zA-Z0-9-]*\])?(?P[<>=!~]+[0-9]*(?:\.[0-9]*)?(?:\.[0-9]*)?)?", setup_py_content ) # If there is no airbyte-cdk dependency, add the version - if airbyte_cdk_dependency is not None: - new_version = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={self.new_version}" - return og_setup_py_content.replace(airbyte_cdk_dependency.group(), new_version) - else: + if airbyte_cdk_dependency is None: raise ValueError("Could not find airbyte-cdk dependency in setup.py") + if self.new_version == "latest": + new_version = cdk_helpers.get_latest_python_cdk_version() + else: + new_version = self.new_version + + new_version_str = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={new_version}" + updated_setup_py = setup_py_content.replace(airbyte_cdk_dependency.group(), new_version_str) + + return og_connector_dir.with_new_file("setup.py", updated_setup_py) + async def run_connector_cdk_upgrade_pipeline( context: ConnectorContext, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py new file mode 100644 index 00000000000000..72dbe53b170f86 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format commands. +""" +from __future__ import annotations + +import asyncclick as click +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.lazy_group import LazyGroup +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context + + +@click.group( + name="poetry", + help="Commands related to running poetry commands.", + cls=LazyGroup, + lazy_subcommands={ + "publish": "pipelines.airbyte_ci.poetry.publish.commands.publish", + }, +) +@click.option( + "--package-path", + help="The path to publish", + type=click.STRING, + required=True, +) +@click_merge_args_into_context_obj +@pass_pipeline_context +@click_ignore_unused_kwargs +async def poetry(pipeline_context: ClickPipelineContext) -> None: + pass diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py new file mode 100644 index 00000000000000..29785f8312661f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format commands. +""" +from __future__ import annotations + +from typing import Optional + +import asyncclick as click +from packaging import version +from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry +from pipelines.cli.confirm_prompt import confirm +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.contexts.python_registry_publish import PythonRegistryPublishContext +from pipelines.models.steps import StepStatus + + +async def _has_metadata_yaml(context: PythonRegistryPublishContext) -> bool: + dir_to_publish = context.get_repo_dir(context.package_path) + return "metadata.yaml" in await dir_to_publish.entries() + + +def _validate_python_version(_ctx: dict, _param: dict, value: Optional[str]) -> Optional[str]: + """ + Check if an given version is valid. + """ + if value is None: + return value + try: + version.Version(value) + return value + except version.InvalidVersion: + raise click.BadParameter(f"Version {value} is not a valid version.") + + +@click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to a registry.") +@click.option( + "--python-registry-token", + help="Access token", + type=click.STRING, + required=True, + envvar="PYTHON_REGISTRY_TOKEN", +) +@click.option( + "--registry-url", + help="Which registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", + type=click.STRING, + default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, +) +@click.option( + "--publish-name", + help="The name of the package to publish. If not set, the name will be inferred from the pyproject.toml file of the package.", + type=click.STRING, +) +@click.option( + "--publish-version", + help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.", + type=click.STRING, + callback=_validate_python_version, +) +@pass_pipeline_context +@click.pass_context +async def publish( + ctx: click.Context, + click_pipeline_context: ClickPipelineContext, + python_registry_token: str, + registry_url: str, + publish_name: Optional[str], + publish_version: Optional[str], +) -> bool: + context = PythonRegistryPublishContext( + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + python_registry_token=python_registry_token, + registry=registry_url, + package_path=ctx.obj["package_path"], + package_name=publish_name, + version=publish_version, + ) + + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to python registry") + context.dagger_client = dagger_client + + if await _has_metadata_yaml(context): + confirm( + "It looks like you are trying to publish a connector. In most cases, the `connectors` command group should be used instead. Do you want to continue?", + abort=True, + ) + + publish_result = await PublishToPythonRegistry(context).run() + + return publish_result.status is StepStatus.SUCCESS diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py new file mode 100644 index 00000000000000..aec2e30bb3da2c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py @@ -0,0 +1,165 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import configparser +import io +import uuid +from enum import Enum, auto +from typing import Dict, Optional + +import tomli +import tomli_w +from dagger import Container, Directory +from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH +from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext +from pipelines.models.steps import Step, StepResult + + +class PackageType(Enum): + POETRY = auto() + PIP = auto() + + +class PublishToPythonRegistry(Step): + context: PythonRegistryPublishContext + title = "Publish package to python registry" + + def _get_base_container(self) -> Container: + return with_poetry(self.context) + + async def _get_package_metadata_from_pyproject_toml(self, package_dir_to_publish: Directory) -> Optional[PythonPackageMetadata]: + pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + try: + return PythonPackageMetadata(contents["tool"]["poetry"]["name"], contents["tool"]["poetry"]["version"]) + except KeyError: + return None + + async def _get_package_type(self, package_dir_to_publish: Directory) -> Optional[PackageType]: + files = await package_dir_to_publish.entries() + has_pyproject_toml = PYPROJECT_TOML_FILE_PATH in files + has_setup_py = SETUP_PY_FILE_PATH in files + if has_pyproject_toml: + return PackageType.POETRY + elif has_setup_py: + return PackageType.PIP + else: + return None + + async def _run(self) -> StepResult: + package_dir_to_publish = await self.context.get_repo_dir(self.context.package_path) + package_type = await self._get_package_type(package_dir_to_publish) + + if not package_type: + return self.skip("Connector does not have a pyproject.toml file or setup.py file, skipping.") + + result = await self._ensure_package_name_and_version(package_dir_to_publish, package_type) + if result: + return result + + self.logger.info( + f"Uploading package {self.context.package_metadata.name} version {self.context.package_metadata.version} to {self.context.registry}..." + ) + + return await self._publish(package_dir_to_publish, package_type) + + async def _ensure_package_name_and_version(self, package_dir_to_publish: Directory, package_type: PackageType) -> Optional[StepResult]: + """ + Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set. + Setup.py packages need to set package name and version as parameter. + + Returns None if package name and version are set, otherwise a StepResult with a skip message. + """ + if self.context.package_metadata.name and self.context.package_metadata.version: + return None + + if package_type is not PackageType.POETRY: + return self.skip("Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping.") + + inferred_package_metadata = await self._get_package_metadata_from_pyproject_toml(package_dir_to_publish) + + if not inferred_package_metadata: + return self.skip( + "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." + ) + + if not self.context.package_metadata.name: + self.context.package_metadata.name = inferred_package_metadata.name + if not self.context.package_metadata.version: + self.context.package_metadata.version = inferred_package_metadata.version + + return None + + async def _publish(self, package_dir_to_publish: Directory, package_type: PackageType) -> StepResult: + if package_type is PackageType.PIP: + return await self._pip_publish(package_dir_to_publish) + else: + return await self._poetry_publish(package_dir_to_publish) + + async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult: + python_registry_token = self.context.dagger_client.set_secret("python_registry_token", self.context.python_registry_token) + pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + # make sure package name and version are set to the configured one + contents["tool"]["poetry"]["name"] = self.context.package_metadata.name + contents["tool"]["poetry"]["version"] = self.context.package_metadata.version + # enforce consistent author + contents["tool"]["poetry"]["authors"] = ["Airbyte "] + poetry_publish = ( + self._get_base_container() + .with_secret_variable("PYTHON_REGISTRY_TOKEN", python_registry_token) + .with_directory("package", package_dir_to_publish) + .with_workdir("package") + .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents)) + .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry]) + .with_exec(sh_dash_c(["poetry config pypi-token.mypypi $PYTHON_REGISTRY_TOKEN"])) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_exec(sh_dash_c(["poetry publish --build --repository mypypi -vvv --no-interaction"])) + ) + + return await self.get_step_result(poetry_publish) + + async def _pip_publish(self, package_dir_to_publish: Directory) -> StepResult: + files = await package_dir_to_publish.entries() + pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") + pypi_password = self.context.dagger_client.set_secret("pypi_password", self.context.python_registry_token) + metadata: Dict[str, str] = { + "name": str(self.context.package_metadata.name), + "version": str(self.context.package_metadata.version), + # Enforce consistent author + "author": "Airbyte", + "author_email": "contact@airbyte.io", + } + if "README.md" in files: + metadata["long_description"] = await package_dir_to_publish.file("README.md").contents() + metadata["long_description_content_type"] = "text/markdown" + + config = configparser.ConfigParser() + config["metadata"] = metadata + + setup_cfg_io = io.StringIO() + config.write(setup_cfg_io) + setup_cfg = setup_cfg_io.getvalue() + + twine_upload = ( + self._get_base_container() + .with_exec(sh_dash_c(["apt-get update", "apt-get install -y twine"])) + .with_directory("package", package_dir_to_publish) + .with_workdir("package") + # clear out setup.py metadata so setup.cfg is used + .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", SETUP_PY_FILE_PATH]) + .with_new_file("setup.cfg", contents=setup_cfg) + .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) + .with_exec(["python", SETUP_PY_FILE_PATH, "sdist", "bdist_wheel"]) + .with_secret_variable("TWINE_USERNAME", pypi_username) + .with_secret_variable("TWINE_PASSWORD", pypi_password) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_exec(["twine", "upload", "--verbose", "--repository-url", self.context.registry, "dist/*"]) + ) + + return await self.get_step_result(twine_upload) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 8779fee5eab1b5..f0a9a526f66f4d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -130,6 +130,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: help="Airbyte CI top-level command group.", lazy_subcommands={ "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", + "poetry": "pipelines.airbyte_ci.poetry.commands.poetry", "format": "pipelines.airbyte_ci.format.commands.format_code", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 851578cc7a0cac..20bb7bd55211d9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -60,6 +60,8 @@ POETRY_CACHE_PATH = "/root/.cache/pypoetry" STORAGE_DRIVER = "fuse-overlayfs" TAILSCALE_AUTH_KEY = os.getenv("TAILSCALE_AUTH_KEY") +SETUP_PY_FILE_PATH = "setup.py" +DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://pypi.org/simple" class CIContext(str, Enum): diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index e13ff7abd8a66a..2a943ddd3dd0c0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -19,7 +19,18 @@ from pipelines.airbyte_ci.connectors.context import ConnectorContext -async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) -> list[str]: +# List of overrides for the secrets masking logic. +# These keywords may have been marked as secrets, perhaps somewhat aggressively. +# Masking them, however, is annoying and pointless. +# This list should be extended (carefully) as needed. +NOT_REALLY_SECRETS = { + "admin", + "airbyte", + "host", +} + + +async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container, connector_technical_name: str) -> list[str]: """This function will print the secrets to mask in the GitHub actions logs with the ::add-mask:: prefix. We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger, And GHA will only interpret lines starting with ::add-mask:: as secrets to mask. @@ -27,6 +38,9 @@ async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) secrets_to_mask = [] if secrets_to_mask_file := await get_file_contents(ci_credentials_with_downloaded_secrets, "/tmp/secrets_to_mask.txt"): for secret_to_mask in secrets_to_mask_file.splitlines(): + if secret_to_mask in NOT_REALLY_SECRETS or secret_to_mask in connector_technical_name: + # Don't mask secrets which are also common words or connector name. + continue # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked print(f"::add-mask::{secret_to_mask}") @@ -59,7 +73,7 @@ async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = " ) # We don't want to print secrets in the logs when running locally. if context.is_ci: - context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets) + context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets, context.connector.technical_name) connector_secrets = {} for secret_file in await with_downloaded_secrets.directory(secrets_path).entries(): secret_plaintext = await with_downloaded_secrets.directory(secrets_path).file(secret_file).contents() @@ -147,8 +161,15 @@ async def mounted_connector_secrets(context: ConnectorContext, secret_directory_ fn (Callable[[Container], Container]): A function to pass as argument to the connector container's with_ method. """ connector_secrets = await context.get_connector_secrets() + java_log_scrub_pattern_secret = context.java_log_scrub_pattern_secret def with_secrets_mounted_as_dagger_secrets(container: Container) -> Container: + if java_log_scrub_pattern_secret: + # This LOG_SCRUB_PATTERN environment variable is used by our log4j test configuration + # to scrub secrets from the log messages. Although we already ensure that github scrubs them + # from its runner logs, this is required to prevent the secrets from leaking into gradle scans, + # test reports or any other build artifacts generated by a java connector test. + container = container.with_secret_variable("LOG_SCRUB_PATTERN", java_log_scrub_pattern_secret) container = container.with_exec(["mkdir", "-p", secret_directory_path], skip_entrypoint=True) for secret_file_name, secret in connector_secrets.items(): container = container.with_mounted_secret(f"{secret_directory_path}/{secret_file_name}", secret) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py new file mode 100644 index 00000000000000..726fdd98bf232b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import re + +import requests # type: ignore +from dagger import Directory + + +def get_latest_python_cdk_version() -> str: + """ + Get the latest version of airbyte-cdk from pypi + """ + cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" + response = requests.get(cdk_pypi_url) + response.raise_for_status() + package_info = response.json() + return package_info["info"]["version"] + + +async def get_latest_java_cdk_version(repo_dir: Directory) -> str: + version_file_content = await repo_dir.file("airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties").contents() + match = re.search(r"version *= *(?P[0-9]*\.[0-9]*\.[0-9]*)", version_file_content) + if match: + return match.group("version") + raise ValueError("Could not find version in version.properties") diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py new file mode 100644 index 00000000000000..e8b1d0ed0dad2f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Optional +from urllib.parse import urlparse + +import requests # type: ignore + + +def is_package_published(package_name: Optional[str], version: Optional[str], registry_url: str) -> bool: + """ + Check if a package with a specific version is published on PyPI or Test PyPI. + + :param package_name: The name of the package to check. + :param version: The version of the package. + :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI. + :return: True if the package is found with the specified version, False otherwise. + """ + if not package_name or not version: + return False + + parsed_registry_url = urlparse(registry_url) + base_url = f"{parsed_registry_url.scheme}://{parsed_registry_url.netloc}" + + url = f"{base_url}/{package_name}/{version}/json" + + response = requests.get(url) + return response.status_code == 200 diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py index affc981a60de0b..619be4278b5752 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py @@ -4,11 +4,11 @@ import json -import requests # type: ignore +import requests from pipelines import main_logger -def send_message_to_webhook(message: str, channel: str, webhook: str) -> dict: +def send_message_to_webhook(message: str, channel: str, webhook: str) -> requests.Response: payload = {"channel": f"#{channel}", "username": "Connectors CI/CD Bot", "text": message} response = requests.post(webhook, data={"payload": json.dumps(payload)}) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index c796e9f6e43b05..e275b42610bfe2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -11,6 +11,7 @@ import re import sys import unicodedata +import xml.sax.saxutils from io import TextIOWrapper from pathlib import Path from typing import TYPE_CHECKING @@ -326,3 +327,29 @@ def fail_if_missing_docker_hub_creds(ctx: click.Context) -> None: raise click.UsageError( "You need to be logged to DockerHub registry to run this command. Please set DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD environment variables." ) + + +def java_log_scrub_pattern(secrets_to_mask: List[str]) -> str: + """Transforms a list of secrets into a LOG_SCRUB_PATTERN env var value for our log4j test configuration.""" + # Build a regex pattern that matches any of the secrets to mask. + regex_pattern = "|".join(map(re.escape, secrets_to_mask)) + # Now, make this string safe to consume by the log4j configuration. + # Its parser is XML-based so the pattern needs to be escaped again, and carefully. + return xml.sax.saxutils.escape( + regex_pattern, + # Annoyingly, the log4j properties file parser is quite brittle when it comes to + # handling log message patterns. In our case the env var is injected like this: + # + # ${env:LOG_SCRUB_PATTERN:-defaultvalue} + # + # We must avoid confusing the parser with curly braces or colons otherwise the + # printed log messages will just consist of `%replace`. + { + "\t": " ", + "'": "'", + '"': """, + "{": "{", + "}": "}", + ":": ":", + }, + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index fd565105c70ce2..e3182bbd0377a4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -5,6 +5,7 @@ import io import sys import tempfile +from pathlib import Path from typing import Any, Callable, Dict, Optional, TextIO, Tuple import anyio @@ -108,13 +109,13 @@ def get_log_output(self) -> TextIO: log_output, self._click_context().obj["dagger_logs_path"] = self._create_dagger_client_log_file() return log_output - def _create_dagger_client_log_file(self) -> Tuple[TextIO, str]: + def _create_dagger_client_log_file(self) -> Tuple[TextIO, Path]: """ Create the dagger client log file. """ dagger_logs_file_descriptor, dagger_logs_temp_file_path = tempfile.mkstemp(dir="/tmp", prefix="dagger_client_", suffix=".log") main_logger.info(f"Dagger client logs stored in {dagger_logs_temp_file_path}") - return io.TextIOWrapper(io.FileIO(dagger_logs_file_descriptor, "w+")), dagger_logs_temp_file_path + return io.TextIOWrapper(io.FileIO(dagger_logs_file_descriptor, "w+")), Path(dagger_logs_temp_file_path) # Create @pass_pipeline_context decorator for use in click commands diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py index 0d2431560145fb..b1cf2159091213 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py @@ -22,7 +22,7 @@ from pipelines.helpers.gcs import sanitize_gcs_credentials from pipelines.helpers.github import update_commit_status_check from pipelines.helpers.slack import send_message_to_webhook -from pipelines.helpers.utils import AIRBYTE_REPO_URL +from pipelines.helpers.utils import AIRBYTE_REPO_URL, java_log_scrub_pattern from pipelines.models.reports import Report if TYPE_CHECKING: @@ -166,6 +166,12 @@ def ci_github_access_token_secret(self) -> Secret: assert self.ci_github_access_token is not None, "The ci_github_access_token was not set on this PipelineContext." return self.dagger_client.set_secret("ci_github_access_token", self.ci_github_access_token) + @property + def java_log_scrub_pattern_secret(self) -> Optional[Secret]: + if not self.secrets_to_mask: + return None + return self.dagger_client.set_secret("log_scrub_pattern", java_log_scrub_pattern(self.secrets_to_mask)) + @property def github_commit_status(self) -> dict: """Build a dictionary used as kwargs to the update_commit_status_check function.""" diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py new file mode 100644 index 00000000000000..ee45760d2f9d9f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +from dataclasses import dataclass +from datetime import datetime +from typing import Optional, Type + +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL + + +@dataclass +class PythonPackageMetadata: + name: Optional[str] + version: Optional[str] + + +class PythonRegistryPublishContext(PipelineContext): + def __init__( + self, + python_registry_token: str, + package_path: str, + report_output_prefix: str, + is_local: bool, + git_branch: str, + git_revision: str, + ci_report_bucket: Optional[str] = None, + registry: str = DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, + gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, + pipeline_start_timestamp: Optional[int] = None, + ci_context: Optional[str] = None, + ci_gcs_credentials: Optional[str] = None, + package_name: Optional[str] = None, + version: Optional[str] = None, + ) -> None: + self.python_registry_token = python_registry_token + self.registry = registry + self.package_path = package_path + self.package_metadata = PythonPackageMetadata(package_name, version) + + pipeline_name = f"Publish PyPI {package_path}" + + super().__init__( + pipeline_name=pipeline_name, + report_output_prefix=report_output_prefix, + ci_report_bucket=ci_report_bucket, + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ci_gcs_credentials=ci_gcs_credentials, + ) + + @classmethod + async def from_publish_connector_context( + cls: Type["PythonRegistryPublishContext"], connector_context: PublishConnectorContext + ) -> Optional["PythonRegistryPublishContext"]: + """ + Create a PythonRegistryPublishContext from a ConnectorContext. + + The metadata of the connector is read from the current workdir to capture changes that are not yet published. + If pypi is not enabled, this will return None. + """ + + current_metadata = connector_context.connector.metadata + connector_context.logger.info(f"Current metadata: {str(current_metadata)}") + if ( + "remoteRegistries" not in current_metadata + or "pypi" not in current_metadata["remoteRegistries"] + or not current_metadata["remoteRegistries"]["pypi"]["enabled"] + ): + return None + + version = current_metadata["dockerImageTag"] + if connector_context.pre_release: + # use current date as pre-release version + # we can't use the git revision because not all python registries allow local version identifiers. Public version identifiers must conform to PEP 440 and only allow digits. + release_candidate_tag = datetime.now().strftime("%Y%m%d%H%M") + version = f"{version}.dev{release_candidate_tag}" + + pypi_context = cls( + python_registry_token=os.environ["PYTHON_REGISTRY_TOKEN"], + registry="https://test.pypi.org/legacy/", # TODO: go live + package_path=str(connector_context.connector.code_directory), + package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], + version=version, + ci_report_bucket=connector_context.ci_report_bucket, + report_output_prefix=connector_context.report_output_prefix, + is_local=connector_context.is_local, + git_branch=connector_context.git_branch, + git_revision=connector_context.git_revision, + gha_workflow_run_url=connector_context.gha_workflow_run_url, + dagger_logs_url=connector_context.dagger_logs_url, + pipeline_start_timestamp=connector_context.pipeline_start_timestamp, + ci_context=connector_context.ci_context, + ci_gcs_credentials=connector_context.ci_gcs_credentials, + ) + pypi_context.dagger_client = connector_context.dagger_client + return pypi_context diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index e97b0e920b7285..ded75cac60a8c9 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-connectors-base-images" @@ -1150,16 +1150,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1987,7 +1977,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1995,15 +1984,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2020,7 +2002,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2028,7 +2009,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2299,6 +2279,42 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tomli-w" +version = "1.0.0" +description = "A lil' TOML writer" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli_w-1.0.0-py3-none-any.whl", hash = "sha256:9f2a07e8be30a0729e533ec968016807069991ae2fd921a78d42f429ae5f4463"}, + {file = "tomli_w-1.0.0.tar.gz", hash = "sha256:f463434305e0336248cac9c2dc8076b707d8a12d019dd349f5c1e382dd1ae1b9"}, +] + +[[package]] +name = "types-requests" +version = "2.28.2" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.28.2.tar.gz", hash = "sha256:398f88cd9302c796cb63d1021af2a1fb7ae507741a3d508edf8e0746d8c16a04"}, + {file = "types_requests-2.28.2-py3-none-any.whl", hash = "sha256:c164696bfdce0123901165c5f097a6cc4f6326268c65815d4b6a57eacfec5e81"}, +] + +[package.dependencies] +types-urllib3 = "<1.27" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -2560,4 +2576,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "0c7f7c9e18637d2cf9402f22c71502916cd4a1938111dd78eb7874f2c061c1fe" +content-hash = "de011a00b912c9acd6d4f202c7cf54308010efc14a214ee25608712851306aa9" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 226fccfde693e7..25012ff9d0ce22 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.5.3" +version = "3.7.2" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] @@ -27,6 +27,9 @@ segment-analytics-python = "^2.2.3" pygit2 = "^1.13.1" asyncclick = "^8.1.3.4" certifi = "^2023.11.17" +tomli = "^2.0.1" +tomli-w = "^1.0.0" +types-requests = "2.28.2" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2" diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py index f63bd000bbc765..13ce951d64fe88 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py @@ -193,6 +193,12 @@ def test_sh_dash_c(): assert utils.sh_dash_c([]) == ["sh", "-c", "set -o xtrace"] +def test_java_log_scrub_pattern(): + assert utils.java_log_scrub_pattern([]) == "" + assert utils.java_log_scrub_pattern(["foo", "bar"]) == "foo|bar" + assert utils.java_log_scrub_pattern(["|", "'\"{}\t[]<>&"]) == "\\||'"\\{\\}\\ \\[\\]<>\\&" + + @pytest.mark.anyio @pytest.mark.parametrize("tar_file_name", [None, "custom_tar_name.tar"]) async def test_export_container_to_tarball(mocker, dagger_client, tmp_path, tar_file_name): diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py new file mode 100644 index 00000000000000..ee345b41838092 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest +import requests +from dagger import Client, Platform +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def context(dagger_client: Client): + context = PythonRegistryPublishContext( + package_path="test", + version="0.2.0", + python_registry_token="test", + package_name="test", + registry="http://local_registry:8080/", + is_local=True, + git_branch="test", + git_revision="test", + report_output_prefix="test", + ci_report_bucket="test", + ) + context.dagger_client = dagger_client + return context + + +@pytest.mark.parametrize( + "package_path, package_name, expected_asset", + [ + pytest.param( + "airbyte-integrations/connectors/source-apify-dataset", + "airbyte-source-apify-dataset", + "airbyte_source_apify_dataset-0.2.0-py3-none-any.whl", + id="setup.py project", + ), + pytest.param( + "airbyte-integrations/connectors/destination-duckdb", + "destination-duckdb", + "destination_duckdb-0.2.0-py3-none-any.whl", + id="poetry project", + ), + ], +) +async def test_run_poetry_publish(context: PythonRegistryPublishContext, package_path: str, package_name: str, expected_asset: str): + context.package_metadata = PythonPackageMetadata(package_name, "0.2.0") + context.package_path = package_path + pypi_registry = ( + # need to use linux/amd64 because the pypiserver image is only available for that platform + context.dagger_client.container(platform=Platform("linux/amd64")) + .from_("pypiserver/pypiserver:v2.0.1") + .with_exec(["run", "-P", ".", "-a", "."]) + .with_exposed_port(8080) + .as_service() + ) + + base_container = with_poetry(context).with_service_binding("local_registry", pypi_registry) + step = publish_pipeline.PublishToPythonRegistry(context) + step._get_base_container = MagicMock(return_value=base_container) + step_result = await step.run() + assert step_result.status == StepStatus.SUCCESS + + # Query the registry to check that the package was published + tunnel = await context.dagger_client.host().tunnel(pypi_registry).start() + endpoint = await tunnel.endpoint(scheme="http") + list_url = f"{endpoint}/simple/" + list_response = requests.get(list_url) + assert list_response.status_code == 200 + assert package_name in list_response.text + url = f"{endpoint}/simple/{package_name}" + response = requests.get(url) + assert response.status_code == 200 + assert expected_asset in response.text diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index b7fe7d764d5fb4..ce8913e648abdd 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -2,8 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import json +import os import random from typing import List +from unittest.mock import patch import anyio import pytest @@ -153,6 +155,7 @@ def test_parse_spec_output_no_spec(self, publish_context): (publish_pipeline, "PushConnectorImageToRegistry"), (publish_pipeline, "PullConnectorImageFromRegistry"), (publish_pipeline.steps, "run_connector_build"), + (publish_pipeline, "CheckPythonRegistryPackageDoesNotExist"), ] @@ -333,3 +336,73 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( publish_pipeline.PullConnectorImageFromRegistry.return_value.run.assert_not_called() publish_pipeline.UploadSpecToCache.return_value.run.assert_not_called() publish_pipeline.MetadataUpload.return_value.run.assert_not_called() + + +@pytest.mark.parametrize( + "pypi_enabled, pypi_package_does_not_exist_status, publish_step_status, expect_publish_to_pypi_called, expect_build_connector_called", + [ + pytest.param(True, StepStatus.SUCCESS, StepStatus.SUCCESS, True, True, id="happy_path"), + pytest.param(False, StepStatus.SUCCESS, StepStatus.SUCCESS, False, True, id="pypi_disabled, skip all pypi steps"), + pytest.param(True, StepStatus.SKIPPED, StepStatus.SUCCESS, False, True, id="pypi_package_exists, skip publish_to_pypi"), + pytest.param(True, StepStatus.SUCCESS, StepStatus.FAILURE, True, False, id="publish_step_fails, abort"), + pytest.param(True, StepStatus.FAILURE, StepStatus.FAILURE, False, False, id="pypi_package_does_not_exist_fails, abort"), + ], +) +async def test_run_connector_python_registry_publish_pipeline( + mocker, + pypi_enabled, + pypi_package_does_not_exist_status, + publish_step_status, + expect_publish_to_pypi_called, + expect_build_connector_called, +): + + for module, to_mock in STEPS_TO_PATCH: + mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) + + mocked_publish_to_python_registry = mocker.patch( + "pipelines.airbyte_ci.connectors.publish.pipeline.PublishToPythonRegistry", return_value=mocker.AsyncMock() + ) + + for step in [ + publish_pipeline.MetadataValidation, + publish_pipeline.CheckConnectorImageDoesNotExist, + publish_pipeline.UploadSpecToCache, + publish_pipeline.MetadataUpload, + publish_pipeline.PushConnectorImageToRegistry, + publish_pipeline.PullConnectorImageFromRegistry, + ]: + step.return_value.run.return_value = mocker.Mock(name=f"{step.title}_result", status=StepStatus.SUCCESS) + + mocked_publish_to_python_registry.return_value.run.return_value = mocker.Mock( + name="publish_to_python_registry_result", status=publish_step_status + ) + + publish_pipeline.CheckPythonRegistryPackageDoesNotExist.return_value.run.return_value = mocker.Mock( + name="python_registry_package_does_not_exist_result", status=pypi_package_does_not_exist_status + ) + + context = mocker.MagicMock( + ci_gcs_credentials="", + pre_release=False, + connector=mocker.MagicMock( + code_directory="path/to/connector", + metadata={"dockerImageTag": "1.2.3", "remoteRegistries": {"pypi": {"enabled": pypi_enabled, "packageName": "test"}}}, + ), + ) + semaphore = anyio.Semaphore(1) + with patch.dict(os.environ, {"PYTHON_REGISTRY_TOKEN": "test"}): + await publish_pipeline.run_connector_publish_pipeline(context, semaphore) + if expect_publish_to_pypi_called: + mocked_publish_to_python_registry.return_value.run.assert_called_once() + # assert that the first argument passed to mocked_publish_to_pypi contains the things from the context + assert mocked_publish_to_python_registry.call_args.args[0].python_registry_token == "test" + assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.name == "test" + assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.version == "1.2.3" + assert mocked_publish_to_python_registry.call_args.args[0].registry == "https://test.pypi.org/legacy/" + assert mocked_publish_to_python_registry.call_args.args[0].package_path == "path/to/connector" + else: + mocked_publish_to_python_registry.return_value.run.assert_not_called() + + if expect_build_connector_called: + publish_pipeline.steps.run_connector_build.assert_called_once() diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py new file mode 100644 index 00000000000000..cbe91b4df3c1e9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import random +from pathlib import Path +from typing import List +from unittest.mock import AsyncMock, MagicMock + +import anyio +import pytest +from connector_ops.utils import Connector, ConnectorLanguage +from dagger import Directory +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def sample_connector(): + return Connector("source-postgres") + + +def get_sample_build_gradle(airbyte_cdk_version: str, useLocalCdk: str): + return f"""import org.jsonschema2pojo.SourceType + +plugins {{ + id 'application' + id 'airbyte-java-connector' + id "org.jsonschema2pojo" version "1.2.1" +}} + +java {{ + compileJava {{ + options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked" + }} +}} + +airbyteJavaConnector {{ + cdkVersionRequired = '{airbyte_cdk_version}' + features = ['db-sources'] + useLocalCdk = {useLocalCdk} +}} + + +application {{ + mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] +}} +""" + + +@pytest.fixture +def connector_context(sample_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test", + connector=sample_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + +@pytest.mark.parametrize( + "build_gradle_content, expected_build_gradle_content", + [ + (get_sample_build_gradle("1.2.3", "false"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("1.2.3", "true"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("6.6.6", "false"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("6.6.6", "true"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("7.0.0", "false"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("7.0.0", "true"), get_sample_build_gradle("6.6.6", "false")), + ], +) +async def test_run_connector_cdk_upgrade_pipeline( + connector_context: ConnectorContext, build_gradle_content: str, expected_build_gradle_content: str +): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", build_gradle_content) + + # For this test, replace the actual connector dir with an updated version that sets the build.gradle contents + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + # Mock the diff method to record the resulting directory and return a mock to not actually export the diff to the repo + updated_connector_dir.diff = MagicMock(return_value=AsyncMock()) + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.SUCCESS + + # Check that the resulting directory that got passed to the mocked diff method looks as expected + resulting_directory: Directory = await full_og_connector_dir.diff(updated_connector_dir.diff.call_args[0][0]) + files = await resulting_directory.entries() + # validate only build.gradle is changed + assert files == ["build.gradle"] + build_gradle = resulting_directory.file("build.gradle") + actual_build_gradle_content = await build_gradle.contents() + assert expected_build_gradle_content == actual_build_gradle_content + + # Assert that the diff was exported to the repo + assert updated_connector_dir.diff.return_value.export.call_count == 1 + + +async def test_skip_connector_cdk_upgrade_pipeline_on_missing_build_gradle(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.without_file("build.gradle") + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.FAILURE + + +async def test_fail_connector_cdk_upgrade_pipeline_on_missing_airbyte_cdk(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", get_sample_build_gradle("abc", "false")) + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.FAILURE diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_python_cdk.py similarity index 93% rename from airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py rename to airbyte-ci/connectors/pipelines/tests/test_upgrade_python_cdk.py index 9cc3d26f9f27b0..67d855a1d91e7c 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_python_cdk.py @@ -69,6 +69,9 @@ def connector_context(sample_connector, dagger_client, current_platform): (get_sample_setup_py("airbyte-cdk==1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")), (get_sample_setup_py("airbyte-cdk>=1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")), (get_sample_setup_py("airbyte-cdk[file-based]>=1.2.3"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")), + (get_sample_setup_py("airbyte-cdk==1.2"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk>=1.2"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk[file-based]>=1.2"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")), ], ) async def test_run_connector_cdk_upgrade_pipeline( diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index 407a1754e01c33..1ef8a0b8017dd2 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 3.3.0 +Add `test_certified_connector_has_allowed_hosts` and `test_certified_connector_has_suggested_streams` tests to the `connector_attribute` test suite + +## 3.2.0 +Add TestBasicRead.test_all_supported_file_types_present, which validates that all supported file types are present in the sandbox account for certified file-based connectors. + +## 3.1.0 +Add TestSpec.test_oauth_is_default_method test with OAuth is default option validation. + ## 3.0.1 Upgrade to Dagger 0.9.6 diff --git a/airbyte-integrations/bases/connector-acceptance-test/README.md b/airbyte-integrations/bases/connector-acceptance-test/README.md index 3b33b204b51248..9ca4822380ed69 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/README.md +++ b/airbyte-integrations/bases/connector-acceptance-test/README.md @@ -1,7 +1,7 @@ # Connector Acceptance Tests (CAT) This package gathers multiple test suites to assess the sanity of any Airbyte connector. It is shipped as a [pytest](https://docs.pytest.org/en/7.1.x/) plugin and relies on pytest to discover, configure and execute tests. -Test-specific documentation can be found [here](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/)). +Test-specific documentation can be found [here](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/). ## Configuration The acceptance tests are configured via the `acceptance-test-config.yml` YAML file, which is passed to the plugin via the `--acceptance-test-config` option. @@ -93,7 +93,7 @@ These iterations are more conveniently achieved by remaining in the current dire 8. Make sure you updated `docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md` according to your changes 9. Update the project changelog `airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md` 10. Open a PR on our GitHub repository -11. This [Github action workflow](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/cat-tests.yml) will be triggered an run the unit tests on your branch. +11. This [GitHub action workflow](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/cat-tests.yml) will be triggered and run the unit tests on your branch. 12. Publish the new acceptance test version if your PR is approved by running `/legacy-publish connector=bases/connector-acceptance-test run-tests=false` in a GitHub comment 13. Merge your PR diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index 045e23671b74c2..f269fe15db0aa1 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -7,7 +7,7 @@ from copy import deepcopy from enum import Enum from pathlib import Path -from typing import Generic, List, Mapping, Optional, Set, TypeVar +from typing import Any, Dict, Generic, List, Mapping, Optional, Set, TypeVar from pydantic import BaseModel, Field, root_validator, validator from pydantic.generics import GenericModel @@ -42,6 +42,17 @@ class BackwardCompatibilityTestsConfig(BaseConfig): ) +class OAuthTestConfig(BaseConfig): + oauth = Field(True, description="Allow source to have another default method that OAuth.") + bypass_reason: Optional[str] = Field(description="Reason why OAuth is not default method.") + + @validator("oauth", always=True) + def validate_oauth(cls, oauth, values): + if oauth is False and not values.get("bypass_reason"): + raise ValueError("Please provide a bypass reason for Auth default method") + return oauth + + class SpecTestConfig(BaseConfig): spec_path: str = spec_path config_path: str = config_path @@ -50,6 +61,7 @@ class SpecTestConfig(BaseConfig): backward_compatibility_tests_config: BackwardCompatibilityTestsConfig = Field( description="Configuration for the backward compatibility tests.", default=BackwardCompatibilityTestsConfig() ) + auth_default_method: Optional[OAuthTestConfig] = Field(description="Auth default method details.") class ConnectionTestConfig(BaseConfig): @@ -82,7 +94,8 @@ class Config: extra_fields: bool = Field(False, description="Allow records to have other fields") exact_order: bool = Field(False, description="Ensure that records produced in exact same order") extra_records: bool = Field( - True, description="Allow connector to produce extra records, but still enforce all records from the expected file to be produced" + True, + description="Allow connector to produce extra records, but still enforce all records from the expected file to be produced", ) @validator("exact_order", always=True) @@ -129,6 +142,43 @@ class NoPrimaryKeyConfiguration(BaseConfig): bypass_reason: Optional[str] = Field(default=None, description="Reason why this stream does not support a primary key") +class AllowedHostsConfiguration(BaseConfig): + bypass_reason: Optional[str] = Field( + default=None, description="Reason why the Metadata `AllowedHosts` check should be skipped for this certified connector." + ) + + +class SuggestedStreamsConfiguration(BaseConfig): + bypass_reason: Optional[str] = Field( + default=None, description="Reason why the Metadata `SuggestedStreams` check should be skipped for this certified connector." + ) + + +class UnsupportedFileTypeConfig(BaseConfig): + extension: str + bypass_reason: Optional[str] = Field(description="Reason why this type is considered unsupported.") + + @validator("extension", always=True) + def extension_properly_formatted(cls, extension: str) -> str: + if not extension.startswith(".") or len(extension) < 2: + raise ValueError("Please provide a valid file extension (e.g. '.csv').") + return extension + + +class FileTypesConfig(BaseConfig): + bypass_reason: Optional[str] = Field(description="Reason why this test is bypassed.") + unsupported_types: Optional[List[UnsupportedFileTypeConfig]] = Field(description="A list of unsupported file types for the source.") + skip_test: Optional[bool] = Field(False, description="Skip file-based connector specific test.") + + @validator("skip_test", always=True) + def no_unsupported_types_when_skip_test(cls, skip_test: bool, values: Dict[str, Any]) -> bool: + if skip_test and values.get("unsupported_types"): + raise ValueError("You can't set 'unsupported_types' if the test is skipped.") + if not skip_test and values.get("bypass_reason") is not None: + raise ValueError("You can't set 'bypass_reason' if the test is not skipped.") + return skip_test + + class BasicReadTestConfig(BaseConfig): config_path: str = config_path deployment_mode: Optional[str] = deployment_mode @@ -146,6 +196,10 @@ class BasicReadTestConfig(BaseConfig): expect_trace_message_on_failure: bool = Field(True, description="Ensure that a trace message is emitted when the connector crashes") timeout_seconds: int = timeout_seconds ignored_fields: Optional[Mapping[str, List[IgnoredFieldsConfiguration]]] = ignored_fields + file_types: Optional[FileTypesConfig] = Field( + default_factory=FileTypesConfig, + description="For file-based connectors, unsupported by source file types can be configured or a test can be skipped at all", + ) class FullRefreshConfig(BaseConfig): @@ -164,7 +218,7 @@ class FullRefreshConfig(BaseConfig): class FutureStateConfig(BaseConfig): future_state_path: Optional[str] = Field(description="Path to a state file with values in far future") - missing_streams: List[EmptyStreamConfiguration] = Field(default=[], description="List of missings streams with valid bypass reasons.") + missing_streams: List[EmptyStreamConfiguration] = Field(default=[], description="List of missing streams with valid bypass reasons.") bypass_reason: Optional[str] @@ -197,6 +251,12 @@ class ConnectorAttributesConfig(BaseConfig): streams_without_primary_key: Optional[List[NoPrimaryKeyConfiguration]] = Field( description="Streams that do not support a primary key such as reports streams" ) + allowed_hosts: Optional[AllowedHostsConfiguration] = Field( + description="Used to bypass checking the `allowedHosts` field in a source's `metadata.yaml` when all external hosts should be reachable." + ) + suggested_streams: Optional[SuggestedStreamsConfiguration] = Field( + description="Used to bypass checking the `suggestedStreams` field in a source's `metadata.yaml` when certified source doesn't have any." + ) class GenericTestConfig(GenericModel, Generic[TestConfigT]): @@ -254,9 +314,9 @@ def migrate_legacy_to_current_config(legacy_config: dict) -> dict: """Convert configuration structure created prior to v0.2.12 into the current structure. e.g. This structure: - {"connector_image": "my-connector-image", "tests": {"spec": [{"spec_path": "my/spec/path.json"}]} + {"connector_image": "my-connector-image", "tests": {"spec": [{"spec_path": "my/spec/path.json"}]}} Gets converted to: - {"connector_image": "my-connector-image", "acceptance_tests": {"spec": {"tests": [{"spec_path": "my/spec/path.json"}]}} + {"connector_image": "my-connector-image", "acceptance_tests": {"spec": {"tests": [{"spec_path": "my/spec/path.json"}]}}} Args: legacy_config (dict): A legacy configuration @@ -301,7 +361,7 @@ def legacy_format_adapter(cls, values: dict) -> dict: dict: The migrated configuration if needed. """ if ALLOW_LEGACY_CONFIG and cls.is_legacy(values): - logging.warn("The acceptance-test-config.yml file is in a legacy format. Please migrate to the latest format.") + logging.warning("The acceptance-test-config.yml file is in a legacy format. Please migrate to the latest format.") return cls.migrate_legacy_to_current_config(values) else: return values diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py index 7ced702152db6c..3dbac190bc8cd3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py @@ -9,6 +9,7 @@ from collections import Counter, defaultdict from functools import reduce from logging import Logger +from os.path import splitext from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Set, Tuple from xmlrpc.client import Boolean @@ -29,6 +30,7 @@ ) from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import ( + AllowedHostsConfiguration, BasicReadTestConfig, Config, ConnectionTestConfig, @@ -39,6 +41,7 @@ IgnoredFieldsConfiguration, NoPrimaryKeyConfiguration, SpecTestConfig, + UnsupportedFileTypeConfig, ) from connector_acceptance_test.utils import ConnectorRunner, SecretDict, delete_fields, filter_output, make_hashable, verify_records_schema from connector_acceptance_test.utils.backward_compatibility import CatalogDiffChecker, SpecDiffChecker, validate_previous_configs @@ -120,6 +123,12 @@ async def skip_backward_compatibility_tests_fixture( pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") return False + @pytest.fixture(name="skip_oauth_default_method_test") + def skip_oauth_default_method_test_fixture(self, inputs: SpecTestConfig): + if inputs.auth_default_method and not inputs.auth_default_method.oauth: + pytest.skip(f"Skipping OAuth is default method test: {inputs.auth_default_method.bypass_reason}") + return False + def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict): """Check that config matches the actual schema from the spec call""" # Getting rid of technical variables that start with an underscore @@ -521,6 +530,29 @@ def test_oauth_flow_parameters(self, actual_connector_spec: ConnectorSpecificati diff = paths_to_validate - set(get_expected_schema_structure(spec_schema)) assert diff == set(), f"Specified oauth fields are missed from spec schema: {diff}" + def test_oauth_is_default_method(self, skip_oauth_default_method_test: bool, actual_connector_spec: ConnectorSpecification): + """ + OAuth is default check. + If credentials do have oneOf: we check that the OAuth is listed at first. + If there is no oneOf and Oauth: OAuth is only option to authenticate the source and no check is needed. + """ + advanced_auth = actual_connector_spec.advanced_auth + if not advanced_auth: + pytest.skip("Source does not have OAuth method.") + + spec_schema = actual_connector_spec.connectionSpecification + credentials = advanced_auth.predicate_key[0] + try: + one_of_default_method = dpath.util.get(spec_schema, f"/**/{credentials}/oneOf/0") + except KeyError as e: # Key Error when oneOf is not in credentials object + pytest.skip("Credentials object does not have oneOf option.") + + path_in_credentials = "/".join(advanced_auth.predicate_key[1:]) + auth_method_predicate_const = dpath.util.get(one_of_default_method, f"/**/{path_in_credentials}/const") + assert ( + auth_method_predicate_const == advanced_auth.predicate_value + ), f"Oauth method should be a default option. Current default method is {auth_method_predicate_const}." + @pytest.mark.default_timeout(ONE_MINUTE) @pytest.mark.backward_compatibility def test_backward_compatibility( @@ -983,10 +1015,12 @@ def configured_catalog_fixture( else: return build_configured_catalog_from_custom_catalog(configured_catalog_path, discovered_catalog) + _file_types: Set[str] = set() + async def test_read( self, - connector_config, - configured_catalog, + connector_config: SecretDict, + configured_catalog: ConfiguredAirbyteCatalog, expect_records_config: ExpectedRecordsConfig, should_validate_schema: Boolean, should_validate_data_points: Boolean, @@ -995,11 +1029,15 @@ async def test_read( ignored_fields: Optional[Mapping[str, List[IgnoredFieldsConfiguration]]], expected_records_by_stream: MutableMapping[str, List[MutableMapping]], docker_runner: ConnectorRunner, - detailed_logger, + detailed_logger: Logger, + certified_file_based_connector: bool, ): output = await docker_runner.call_read(connector_config, configured_catalog) records = [message.record for message in filter_output(output, Type.RECORD)] + if certified_file_based_connector: + self._file_types.update(self._get_actual_file_types(records)) + assert records, "At least one record should be read using provided catalog" if should_validate_schema: @@ -1144,10 +1182,61 @@ def group_by_stream(records: List[AirbyteRecordMessage]) -> MutableMapping[str, return result + @pytest.fixture(name="certified_file_based_connector") + def is_certified_file_based_connector(self, connector_metadata: Dict[str, Any]) -> bool: + metadata = connector_metadata.get("data", {}) + + # connector subtype is specified in data.connectorSubtype field + file_based_connector = metadata.get("connectorSubtype") == "file" + # a certified connector has ab_internal.ql value >= 400 + certified_connector = metadata.get("ab_internal", {}).get("ql", 0) >= 400 + + return file_based_connector and certified_connector + + @staticmethod + def _get_file_extension(file_name: str) -> str: + _, file_extension = splitext(file_name) + return file_extension.casefold() + + def _get_actual_file_types(self, records: List[AirbyteRecordMessage]) -> Set[str]: + return {self._get_file_extension(record.data.get("_ab_source_file_url", "")) for record in records} + + @staticmethod + def _get_unsupported_file_types(config: List[UnsupportedFileTypeConfig]) -> Set[str]: + return {t.extension.casefold() for t in config} + + async def test_all_supported_file_types_present(self, certified_file_based_connector: bool, inputs: BasicReadTestConfig): + if not certified_file_based_connector or inputs.file_types.skip_test: + reason = ( + "Skipping the test for supported file types" + f"{' as it is only applicable for certified file-based connectors' if not certified_file_based_connector else ''}." + ) + pytest.skip(reason) + + structured_types = {".avro", ".csv", ".jsonl", ".parquet"} + unstructured_types = {".pdf", ".doc", ".docx", ".ppt", ".pptx", ".md"} + + if inputs.file_types.unsupported_types: + unsupported_file_types = self._get_unsupported_file_types(inputs.file_types.unsupported_types) + structured_types.difference_update(unsupported_file_types) + unstructured_types.difference_update(unsupported_file_types) + + missing_structured_types = structured_types - self._file_types + missing_unstructured_types = unstructured_types - self._file_types + + # all structured and at least one of unstructured supported file types should be present + assert not missing_structured_types and len(missing_unstructured_types) != len(unstructured_types), ( + f"Please make sure you added files with the following supported structured types {missing_structured_types} " + f"and at least one with unstructured type {unstructured_types} to the test account " + "or add them to the `file_types -> unsupported_types` list in config." + ) + @pytest.mark.default_timeout(TEN_MINUTES) class TestConnectorAttributes(BaseTest): - MANDATORY_FOR_TEST_STRICTNESS_LEVELS = [] # Used so that this is not part of the mandatory high strictness test suite yet + # Overide from BaseTest! + # Used so that this is not part of the mandatory high strictness test suite yet + MANDATORY_FOR_TEST_STRICTNESS_LEVELS = [] @pytest.fixture(name="operational_certification_test") async def operational_certification_test_fixture(self, connector_metadata: dict) -> bool: @@ -1166,7 +1255,7 @@ def streams_without_primary_key_fixture(self, inputs: ConnectorAttributesConfig) async def test_streams_define_primary_key( self, operational_certification_test, streams_without_primary_key, connector_config, docker_runner: ConnectorRunner - ): + ) -> None: output = await docker_runner.call_discover(config=connector_config) catalog_messages = filter_output(output, Type.CATALOG) streams = catalog_messages[0].catalog.streams @@ -1175,3 +1264,72 @@ async def test_streams_define_primary_key( quoted_missing_primary_keys = {f"'{primary_key}'" for primary_key in missing_primary_keys} assert not missing_primary_keys, f"The following streams {', '.join(quoted_missing_primary_keys)} do not define a primary_key" + + @pytest.fixture(name="allowed_hosts_test") + def allowed_hosts_fixture_test(self, inputs: ConnectorAttributesConfig) -> bool: + allowed_hosts = inputs.allowed_hosts + bypass_reason = allowed_hosts.bypass_reason if allowed_hosts else None + if bypass_reason: + pytest.skip(f"Skipping `metadata.allowedHosts` checks. Reason: {bypass_reason}") + return True + + async def test_certified_connector_has_allowed_hosts( + self, operational_certification_test, allowed_hosts_test, connector_metadata: dict + ) -> None: + """ + Checks whether or not the connector has `allowedHosts` and it's components defined in `metadata.yaml`. + Suitable for certified connectors starting `ql` >= 400. + + Arguments: + :: operational_certification_test -- pytest.fixure defines the connector is suitable for this test or not. + :: connector_metadata -- `metadata.yaml` file content + """ + metadata = connector_metadata.get("data", {}) + + has_allowed_hosts_property = "allowedHosts" in metadata.keys() + assert has_allowed_hosts_property, f"The `allowedHosts` property is missing in `metadata.data` for `metadata.yaml`." + + allowed_hosts = metadata.get("allowedHosts", {}) + has_hosts_property = "hosts" in allowed_hosts.keys() if allowed_hosts else False + assert has_hosts_property, f"The `hosts` property is missing in `metadata.data.allowedHosts` for `metadata.yaml`." + + hosts = allowed_hosts.get("hosts", []) + has_assigned_hosts = len(hosts) > 0 if hosts else False + assert ( + has_assigned_hosts + ), f"The `hosts` empty list is not allowed for `metadata.data.allowedHosts` for certified connectors. Please add `hosts` or define the `allowed_hosts.bypass_reason` in `acceptance-test-config.yaml`." + + @pytest.fixture(name="suggested_streams_test") + def suggested_streams_fixture_test(self, inputs: ConnectorAttributesConfig) -> bool: + suggested_streams = inputs.suggested_streams + bypass_reason = suggested_streams.bypass_reason if suggested_streams else None + if bypass_reason: + pytest.skip(f"Skipping `metadata.suggestedStreams` checks. Reason: {bypass_reason}") + return True + + async def test_certified_connector_has_suggested_streams( + self, operational_certification_test, suggested_streams_test, connector_metadata: dict + ) -> None: + """ + Checks whether or not the connector has `suggestedStreams` and it's components defined in `metadata.yaml`. + Suitable for certified connectors starting `ql` >= 400. + + Arguments: + :: operational_certification_test -- pytest.fixure defines the connector is suitable for this test or not. + :: connector_metadata -- `metadata.yaml` file content + """ + + metadata = connector_metadata.get("data", {}) + + has_suggested_streams_property = "suggestedStreams" in metadata.keys() + assert has_suggested_streams_property, f"The `suggestedStreams` property is missing in `metadata.data` for `metadata.yaml`." + + suggested_streams = metadata.get("suggestedStreams", {}) + has_streams_property = "streams" in suggested_streams.keys() if suggested_streams else False + assert has_streams_property, f"The `streams` property is missing in `metadata.data.suggestedStreams` for `metadata.yaml`." + + streams = suggested_streams.get("streams", []) + has_assigned_suggested_streams = len(streams) > 0 if streams else False + assert ( + has_assigned_suggested_streams + ), f"The `streams` empty list is not allowed for `metadata.data.suggestedStreams` for certified connectors." diff --git a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml index 12f3060287b32c..7be03c21d32dc5 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml +++ b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector-acceptance-test" -version = "3.0.1" +version = "3.2.0" description = "Contains acceptance tests for connectors." authors = ["Airbyte "] license = "MIT" diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py index 2687bfaf510147..5a4e62e5552039 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py @@ -212,3 +212,33 @@ class TestExpectedRecordsConfig: def test_bypass_reason_behavior(self, path, bypass_reason, expectation): with expectation: config.ExpectedRecordsConfig(path=path, bypass_reason=bypass_reason) + + +class TestFileTypesConfig: + @pytest.mark.parametrize( + ("skip_test", "bypass_reason", "unsupported_types", "expectation"), + ( + (True, None, None, does_not_raise()), + (True, None, [config.UnsupportedFileTypeConfig(extension=".csv")], pytest.raises(ValidationError)), + (False, None, None, does_not_raise()), + (False, "bypass_reason", None, pytest.raises(ValidationError)), + (False, "", None, pytest.raises(ValidationError)), + (False, None, [config.UnsupportedFileTypeConfig(extension=".csv")], does_not_raise()), + ), + ) + def test_skip_test_behavior(self, skip_test, bypass_reason, unsupported_types, expectation): + with expectation: + config.FileTypesConfig(skip_test=skip_test, bypass_reason=bypass_reason, unsupported_types=unsupported_types) + + @pytest.mark.parametrize( + ("extension", "expectation"), + ( + (".csv", does_not_raise()), + ("csv", pytest.raises(ValidationError)), + (".", pytest.raises(ValidationError)), + ("", pytest.raises(ValidationError)), + ), + ) + def test_extension_validation(self, extension, expectation): + with expectation: + config.UnsupportedFileTypeConfig(extension=extension) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py index 1730c37a9aff72..7435915090c75e 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py @@ -75,3 +75,103 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea connector_config={}, docker_runner=docker_runner_mock ) + + +@pytest.mark.parametrize( + "metadata_yaml, should_raise_assert_error, expected_error", + [ + pytest.param( + {"data": {"ab_internal": {"ql": 400}}}, + True, + "The `allowedHosts` property is missing in `metadata.data` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "allowedHosts": {}}}, + True, + "The `hosts` property is missing in `metadata.data.allowedHosts` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "allowedHosts": {"hosts": []}}}, + True, + "'The `hosts` empty list is not allowed for `metadata.data.allowedHosts` for certified connectors", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "allowedHosts": {"hosts": ["*.test.com"]}}}, + False, + None, + ), + ], + ids=[ + "No `allowedHosts`", + "Has `allowdHosts` but no `hosts`", + "Has `hosts` but it's empty list", + "Has non-empty `hosts`", + ] +) +async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise_assert_error, expected_error) -> None: + t = test_core.TestConnectorAttributes() + + if should_raise_assert_error: + with pytest.raises(AssertionError) as e: + await t.test_certified_connector_has_allowed_hosts( + operational_certification_test=True, + allowed_hosts_test=True, + connector_metadata=metadata_yaml + ) + assert expected_error in repr(e.value) + else: + await t.test_certified_connector_has_allowed_hosts( + operational_certification_test=True, + allowed_hosts_test=True, + connector_metadata=metadata_yaml + ) + + +@pytest.mark.parametrize( + "metadata_yaml, should_raise_assert_error, expected_error", + [ + pytest.param( + {"data": {"ab_internal": {"ql": 400}}}, + True, + "The `suggestedStreams` property is missing in `metadata.data` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "suggestedStreams": {}}}, + True, + "The `streams` property is missing in `metadata.data.suggestedStreams` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "suggestedStreams": {"streams": []}}}, + True, + "'The `streams` empty list is not allowed for `metadata.data.suggestedStreams` for certified connectors", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "suggestedStreams": {"streams": ["stream_1", "stream_2"]}}}, + False, + None, + ), + ], + ids=[ + "No `suggestedStreams`", + "Has `suggestedStreams` but no `streams`", + "Has `streams` but it's empty list", + "Has non-empty `streams`", + ] +) +async def test_certified_connector_has_suggested_streams(metadata_yaml, should_raise_assert_error, expected_error) -> None: + t = test_core.TestConnectorAttributes() + + if should_raise_assert_error: + with pytest.raises(AssertionError) as e: + await t.test_certified_connector_has_suggested_streams( + operational_certification_test=True, + suggested_streams_test=True, + connector_metadata=metadata_yaml + ) + assert expected_error in repr(e.value) + else: + await t.test_certified_connector_has_suggested_streams( + operational_certification_test=True, + suggested_streams_test=True, + connector_metadata=metadata_yaml + ) \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index ee017ba36c187c..c3552717b22fbb 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -21,7 +21,14 @@ TraceType, Type, ) -from connector_acceptance_test.config import BasicReadTestConfig, Config, ExpectedRecordsConfig, IgnoredFieldsConfiguration +from connector_acceptance_test.config import ( + BasicReadTestConfig, + Config, + ExpectedRecordsConfig, + FileTypesConfig, + IgnoredFieldsConfiguration, + UnsupportedFileTypeConfig, +) from connector_acceptance_test.tests import test_core from jsonschema.exceptions import SchemaError @@ -687,6 +694,7 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor docker_runner=docker_runner_mock, ignored_fields=ignored_fields, detailed_logger=MagicMock(), + certified_file_based_connector=False, ) @@ -741,6 +749,7 @@ async def test_fail_on_extra_columns( docker_runner=docker_runner_mock, ignored_fields=None, detailed_logger=MagicMock(), + certified_file_based_connector=False, ) else: t.test_read( @@ -755,6 +764,7 @@ async def test_fail_on_extra_columns( docker_runner=docker_runner_mock, ignored_fields=None, detailed_logger=MagicMock(), + certified_file_based_connector=False, ) @@ -1319,3 +1329,112 @@ def test_validate_field_appears_at_least_once(records, configured_catalog, expec t._validate_field_appears_at_least_once(records=records, configured_catalog=configured_catalog) else: t._validate_field_appears_at_least_once(records=records, configured_catalog=configured_catalog) + + +@pytest.mark.parametrize( + ("metadata", "expected_file_based"), + ( + ({"data": {"connectorSubtype": "file", "ab_internal": {"ql": 400}}}, True), + ({"data": {"connectorSubtype": "file", "ab_internal": {"ql": 500}}}, True), + ({}, False), + ({"data": {"ab_internal": {}}}, False), + ({"data": {"ab_internal": {"ql": 400}}}, False), + ({"data": {"connectorSubtype": "file"}}, False), + ({"data": {"connectorSubtype": "file", "ab_internal": {"ql": 200}}}, False), + ({"data": {"connectorSubtype": "not_file", "ab_internal": {"ql": 400}}}, False), + ), +) +def test_is_certified_file_based_connector(metadata, expected_file_based): + t = test_core.TestBasicRead() + assert test_core.TestBasicRead.is_certified_file_based_connector.__wrapped__(t, metadata) is expected_file_based + + +@pytest.mark.parametrize( + ("file_name", "expected_extension"), + ( + ("test.csv", ".csv"), + ("test/directory/test.csv", ".csv"), + ("test/directory/test.CSV", ".csv"), + ("test/directory/", ""), + (".bashrc", ""), + ("", ""), + ), +) +def test_get_file_extension(file_name, expected_extension): + t = test_core.TestBasicRead() + assert t._get_file_extension(file_name) == expected_extension + + +@pytest.mark.parametrize( + ("records", "expected_file_types"), + ( + ([], set()), + ( + [ + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test.csv"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test_2.pdf"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test_3.pdf"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test_3.CSV"}, emitted_at=111), + ], + {".csv", ".pdf"}, + ), + ( + [ + AirbyteRecordMessage(stream="stream", data={"field": "value"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": ""}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": ".bashrc"}, emitted_at=111), + ], + {""}, + ), + ), +) +def test_get_actual_file_types(records, expected_file_types): + t = test_core.TestBasicRead() + assert t._get_actual_file_types(records) == expected_file_types + + +@pytest.mark.parametrize( + ("config", "expected_file_types"), + ( + ([], set()), + ([UnsupportedFileTypeConfig(extension=".csv"), UnsupportedFileTypeConfig(extension=".pdf")], {".csv", ".pdf"}), + ([UnsupportedFileTypeConfig(extension=".CSV")], {".csv"}), + ), +) +def test_get_unsupported_file_types(config, expected_file_types): + t = test_core.TestBasicRead() + assert t._get_unsupported_file_types(config) == expected_file_types + + +@pytest.mark.parametrize( + ("is_file_based_connector", "skip_test"), + ((False, True), (False, False), (True, True)), +) +async def test_all_supported_file_types_present_skipped(mocker, is_file_based_connector, skip_test): + mocker.patch.object(test_core.pytest, "skip") + mocker.patch.object(test_core.TestBasicRead, "_file_types", {".avro", ".csv", ".jsonl", ".parquet", ".pdf"}) + + t = test_core.TestBasicRead() + config = BasicReadTestConfig(config_path="config_path", file_types=FileTypesConfig(skip_test=skip_test)) + await t.test_all_supported_file_types_present(is_file_based_connector, config) + test_core.pytest.skip.assert_called_once() + + +@pytest.mark.parametrize( + ("file_types_found", "should_fail"), + ( + ({".avro", ".csv", ".jsonl", ".parquet", ".pdf"}, False), + ({".csv", ".jsonl", ".parquet", ".pdf"}, True), + ({".avro", ".csv", ".jsonl", ".parquet"}, True), + ), +) +async def test_all_supported_file_types_present(mocker, file_types_found, should_fail): + mocker.patch.object(test_core.TestBasicRead, "_file_types", file_types_found) + t = test_core.TestBasicRead() + config = BasicReadTestConfig(config_path="config_path", file_types=FileTypesConfig(skip_test=False)) + + if should_fail: + with pytest.raises(AssertionError) as e: + await t.test_all_supported_file_types_present(certified_file_based_connector=True, inputs=config) + else: + await t.test_all_supported_file_types_present(certified_file_based_connector=True, inputs=config) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py index 3303c2650d1ef6..e4651b20040436 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py @@ -681,7 +681,7 @@ def test_enum_usage(connector_spec, should_fail): }, ), "", - ), + ) ], ) def test_validate_oauth_flow(connector_spec, expected_error): @@ -693,6 +693,314 @@ def test_validate_oauth_flow(connector_spec, expected_error): t.test_oauth_flow_parameters(connector_spec) +@pytest.mark.parametrize( + "connector_spec, expected_error", + [ + # FAIL: OAuth is not default + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": { + "type": "string" + }, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + } + } + }, + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "access_token": { + "type": "string" + }, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + } + } + }, + ] + } + } + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "path_in_connector_config": ["api_url"] + } + } + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + }, + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + }, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } + ), "Oauth method should be a default option. Current default method is access_token." + ), + # SUCCESS: Oauth is default + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": { + "type": "string" + }, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "access_token": { + "type": "string" + }, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + } + } + }, + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + } + } + } + ] + } + } + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "path_in_connector_config": ["api_url"] + } + } + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + }, + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + }, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } + ), "" + ), + # SUCCESS: no advancedAuth specified + (ConnectorSpecification(connectionSpecification={}), ""), + # SUCCESS: only OAuth option to auth + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "object"}, + "credentials": { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "token_expiry_date": {"type": "string", "format": "date-time"}, + }, + }, + }, + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, + }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + }, + }, + ), + "", + ), + # SUCCESS: Credentials object does not have oneOf option. + ( + ConnectorSpecification( + connectionSpecification={"type": "object"}, + advanced_auth={ + "auth_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + }, + ), + "Credentials object does not have oneOf option.", + ), + ] +) +def test_validate_auth_default_method(connector_spec, expected_error): + t = _TestSpec() + if expected_error: + with pytest.raises(AssertionError, match=expected_error): + t.test_oauth_is_default_method(skip_oauth_default_method_test=False, actual_connector_spec=connector_spec) + else: + t.test_oauth_is_default_method(skip_oauth_default_method_test=False, actual_connector_spec=connector_spec) + + @pytest.mark.parametrize( "connector_spec, expectation", [ @@ -740,6 +1048,73 @@ def test_additional_properties_is_true(connector_spec, expectation): ({"type": "object", "properties": {"refresh_token": {"type": "boolean", "airbyte_secret": True}}}, True), ({"type": "object", "properties": {"refresh_token": {"type": ["null", "string"]}}}, True), ({"type": "object", "properties": {"credentials": {"type": "array", "items": {"type": "string"}}}}, True), + ( + { + "type": "object", + "properties": {"credentials": {"type": "object", "properties": { + "client_secret": {"type": "string"}, + "access_token": {"type": "string", "airbyte_secret": True}}}} + }, + True + ), + ( + { + "type": "object", + "properties": {"credentials": {"type": "object", "properties": { + "client_secret": {"type": "string", "airbyte_secret": True}, + "access_token": {"type": "string", "airbyte_secret": True}}}} + }, + False + ), + ( + { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + } + } + }, + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "access_token": { + "type": "string" + }, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + } + } + }, + ] + } + } + }, + True + ), ({"type": "object", "properties": {"auth": {"oneOf": [{"api_token": {"type": "string"}}]}}}, True), ( { @@ -757,7 +1132,7 @@ def test_airbyte_secret(mocker, connector_spec, should_fail): t = _TestSpec() logger = mocker.Mock() t.test_secret_is_properly_marked( - {"connectionSpecification": connector_spec}, logger, ("api_key", "api_token", "refresh_token", "jwt", "credentials") + {"connectionSpecification": connector_spec}, logger, ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret") ) if should_fail: conftest.pytest.fail.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-astra/README.md b/airbyte-integrations/connectors/destination-astra/README.md new file mode 100644 index 00000000000000..2fa995b22593ac --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/README.md @@ -0,0 +1,159 @@ +# Astra Destination + +This is the repository for the Astra destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/astra). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/astra) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_astra/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination astra test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name destination-astra build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-astra:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-astra:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/destination-astra:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-astra:dev spec +```` +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-astra:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-astra:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-astra:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using `airbyte-ci` to run tests +See [airbyte-ci documentation](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command) + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-astra/acceptance-test-config.yml b/airbyte-integrations/connectors/destination-astra/acceptance-test-config.yml new file mode 100644 index 00000000000000..75ab930dc09e3c --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/acceptance-test-config.yml @@ -0,0 +1,5 @@ +acceptance_tests: + spec: + tests: + - spec_path: integration_tests/spec.json +connector_image: airbyte/destination-astra:dev diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/__init__.py b/airbyte-integrations/connectors/destination-astra/destination_astra/__init__.py new file mode 100644 index 00000000000000..1f125a4276a5ce --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationAstra + +__all__ = ["DestinationAstra"] diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/astra_client.py b/airbyte-integrations/connectors/destination-astra/destination_astra/astra_client.py new file mode 100644 index 00000000000000..527c8345daa01f --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/astra_client.py @@ -0,0 +1,152 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from typing import Dict, List, Optional + +import requests +import urllib3 + + +class AstraClient: + def __init__( + self, + astra_endpoint: str, + astra_application_token: str, + keyspace_name: str, + embedding_dim: int, + similarity_function: str, + ): + self.astra_endpoint = astra_endpoint + self.astra_application_token = astra_application_token + self.keyspace_name = keyspace_name + self.embedding_dim = embedding_dim + self.similarity_function = similarity_function + + self.request_base_url = f"{self.astra_endpoint}/api/json/v1/{self.keyspace_name}" + self.request_header = { + "x-cassandra-token": self.astra_application_token, + "Content-Type": "application/json", + } + + def _run_query(self, request_url: str, query: Dict): + try: + response = requests.request("POST", request_url, headers=self.request_header, data=json.dumps(query)) + if response.status_code == 200: + response_dict = json.loads(response.text) + if "errors" in response_dict: + raise Exception(f"Astra DB request error - {response_dict['errors']}") + else: + return response_dict + else: + raise urllib3.exceptions.HTTPError(f"Astra DB not available. Status code: {response.status_code}, {response.text}") + except Exception: + raise + + def find_collections(self, include_detail: bool = True): + query = {"findCollections": {"options": {"explain": include_detail}}} + result = self._run_query(self.request_base_url, query) + + return result["status"]["collections"] + + def find_collection(self, collection_name: str): + collections = self.find_collections(False) + return collection_name in collections + + def create_collection(self, collection_name: str, embedding_dim: Optional[int] = None, similarity_function: Optional[str] = None): + query = { + "createCollection": { + "name": collection_name, + "options": { + "vector": { + "dimension": embedding_dim if embedding_dim is not None else self.embedding_dim, + "metric": similarity_function if similarity_function is not None else self.similarity_function, + } + }, + } + } + result = self._run_query(self.request_base_url, query) + + return True if result["status"]["ok"] == 1 else False + + def delete_collection(self, collection_name: str): + query = {"deleteCollection": {"name": collection_name}} + result = self._run_query(self.request_base_url, query) + + return True if result["status"]["ok"] == 1 else False + + def _build_collection_query(self, collection_name: str): + return f"{self.request_base_url}/{collection_name}" + + def find_documents( + self, + collection_name: str, + filter: Optional[Dict] = None, + vector: Optional[List[float]] = None, + limit: Optional[int] = None, + include_vector: Optional[bool] = None, + include_similarity: Optional[bool] = None, + ) -> List[Dict]: + find_query = {} + + if filter is not None: + find_query["filter"] = filter + + if vector is not None: + find_query["sort"] = {"$vector": vector} + + if include_vector is not None and include_vector == False: + find_query["projection"] = {"$vector": 0} + + if limit is not None: + find_query["options"] = {"limit": limit} + + if include_similarity is not None: + if "options" in find_query: + find_query["options"]["includeSimilarity"] = int(include_similarity) + else: + find_query["options"] = {"includeSimilarity": int(include_similarity)} + + query = {"find": find_query} + result = self._run_query(self._build_collection_query(collection_name), query) + return result["data"]["documents"] + + def insert_document(self, collection_name: str, document: Dict) -> str: + query = {"insertOne": {"document": document}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["insertedIds"][0] + + def insert_documents(self, collection_name: str, documents: List[Dict]) -> List[str]: + query = {"insertMany": {"documents": documents}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["insertedIds"] + + def update_document(self, collection_name: str, filter: Dict, update: Dict, upsert: bool = True) -> Dict: + query = {"findOneAndUpdate": {"filter": filter, "update": update, "options": {"returnDocument": "after", "upsert": upsert}}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"] + + def update_documents(self, collection_name: str, filter: Dict, update: Dict): + query = { + "updateMany": { + "filter": filter, + "update": update, + } + } + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"] + + def count_documents(self, collection_name: str): + query = {"countDocuments": {}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["count"] + + def delete_documents(self, collection_name: str, filter: Dict) -> int: + query = {"deleteMany": {"filter": filter}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["deletedCount"] diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py new file mode 100644 index 00000000000000..7606aab4f1c52f --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel +from pydantic import BaseModel, Field + + +class AstraIndexingModel(BaseModel): + astra_db_app_token: str = Field( + ..., + title="AstraDB Application Token", + airbyte_secret=True, + description="AstraDB Application Token", + ) + astra_db_endpoint: str = Field( + ..., + title="AstraDB Endpoint", + description="AstraDB Endpoint", + pattern="^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\.]*?\\.apps\\.astra\\.datastax\\.com", + examples=["https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com"], + ) + astra_db_keyspace: str = Field(..., title="AstraDB Keyspace", description="Astra DB Keyspace") + collection: str = Field(..., title="AstraDB collection", description="AstraDB collection") + + class Config: + title = "Indexing" + schema_extra = { + "description": "Astra DB gives developers the APIs, real-time data and ecosystem integrations to put accurate RAG and Gen AI apps with fewer hallucinations in production.", + "group": "indexing", + } + + +class ConfigModel(VectorDBConfigModel): + indexing: AstraIndexingModel diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py new file mode 100644 index 00000000000000..6fa1bd7ade5b35 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder, create_from_config +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.writer import Writer +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_astra.config import ConfigModel +from destination_astra.indexer import AstraIndexer + +BATCH_SIZE = 100 + + +class DestinationAstra(Destination): + indexer: Indexer + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = create_from_config(config.embedding, config.processing) + self.indexer = AstraIndexer(config.indexing, self.embedder.embedding_dimensions) + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + writer = Writer( + config_model.processing, self.indexer, self.embedder, batch_size=BATCH_SIZE, omit_raw_text=config_model.omit_raw_text + ) + yield from writer.write(configured_catalog, input_messages) + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + parsed_config = ConfigModel.parse_obj(config) + self._init_indexer(parsed_config) + checks = [self.embedder.check(), self.indexer.check(), DocumentProcessor.check_config(parsed_config.processing)] + errors = [error for error in checks if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/astra", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined] + ) diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py new file mode 100644 index 00000000000000..ee936ae1f5b446 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import uuid +from typing import Optional + +import urllib3 +from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, create_stream_identifier, format_exception +from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog, DestinationSyncMode +from destination_astra.astra_client import AstraClient +from destination_astra.config import AstraIndexingModel + +# do not flood the server with too many connections in parallel +PARALLELISM_LIMIT = 20 + +MAX_METADATA_SIZE = 40_960 - 10_000 + +MAX_IDS_PER_DELETE = 1000 + + +class AstraIndexer(Indexer): + config: AstraIndexingModel + + def __init__(self, config: AstraIndexingModel, embedding_dimensions: int): + super().__init__(config) + + self.client = AstraClient( + config.astra_db_endpoint, config.astra_db_app_token, config.astra_db_keyspace, embedding_dimensions, "cosine" + ) + + self.embedding_dimensions = embedding_dimensions + + def _create_collection(self): + if self.client.find_collection(self.config.collection) is False: + self.client.create_collection(self.config.collection) + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog): + self._create_collection() + for stream in catalog.streams: + if stream.destination_sync_mode == DestinationSyncMode.overwrite: + self.client.delete_documents( + collection_name=self.config.collection, filter={METADATA_STREAM_FIELD: create_stream_identifier(stream.stream)} + ) + + def index(self, document_chunks, namespace, stream): + docs = [] + for i in range(len(document_chunks)): + chunk = document_chunks[i] + metadata = chunk.metadata + if chunk.page_content is not None: + metadata["text"] = chunk.page_content + doc = { + "_id": str(uuid.uuid4()), + "$vector": chunk.embedding, + **metadata, + } + docs.append(doc) + serial_batches = create_chunks(docs, batch_size=PARALLELISM_LIMIT) + + for batch in serial_batches: + results = [chunk for chunk in batch] + self.client.insert_documents(collection_name=self.config.collection, documents=results) + + def delete(self, delete_ids, namespace, stream): + if len(delete_ids) > 0: + self.client.delete_documents(collection_name=self.config.collection, filter={METADATA_RECORD_ID_FIELD: {"$in": delete_ids}}) + + def check(self) -> Optional[str]: + try: + self._create_collection() + collections = self.client.find_collections() + collection = next(filter(lambda f: f["name"] == self.config.collection, collections), None) + if collection is None: + return f"{self.config.collection} collection does not exist." + + actual_dimension = collection["options"]["vector"]["dimension"] + if actual_dimension != self.embedding_dimensions: + return f"Your embedding configuration will produce vectors with dimension {self.embedding_dimensions:d}, but your collection is configured with dimension {actual_dimension:d}. Make sure embedding and indexing configurations match." + except Exception as e: + if isinstance(e, urllib3.exceptions.MaxRetryError): + if "Failed to resolve 'apps.astra.datastax.com'" in str(e.reason): + return "Failed to resolve environment, please check whether the credential is correct." + if isinstance(e, urllib3.exceptions.HTTPError): + return str(e) + + formatted_exception = format_exception(e) + return formatted_exception + return None diff --git a/airbyte-integrations/connectors/destination-astra/icon.svg b/airbyte-integrations/connectors/destination-astra/icon.svg new file mode 100644 index 00000000000000..2d1f6c918ed996 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/icon.svg @@ -0,0 +1,46 @@ + + + + + + + + + + diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py new file mode 100644 index 00000000000000..b9d1aac8ae3c97 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging + +from airbyte_cdk.destinations.vector_db_based.embedder import create_from_config +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status +from destination_astra.astra_client import AstraClient +from destination_astra.config import ConfigModel +from destination_astra.destination import DestinationAstra + + +class AstraIntegrationTest(BaseIntegrationTest): + + def test_check_valid_config(self): + outcome = DestinationAstra().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config(self): + invalid_config = self.config + + invalid_config["embedding"]["openai_key"] = 123 + + outcome = DestinationAstra().check( + logging.getLogger("airbyte"), invalid_config) + assert outcome.status == Status.FAILED + + def test_write(self): + db_config = ConfigModel.parse_obj(self.config) + embedder = create_from_config(db_config.embedding, db_config.processing) + db_creds = db_config.indexing + astra_client = AstraClient( + db_creds.astra_db_endpoint, + db_creds.astra_db_app_token, + db_creds.astra_db_keyspace, + embedder.embedding_dimensions, + "cosine" + ) + + astra_client.delete_documents(collection_name=db_creds.collection, filter={}) + assert astra_client.count_documents(db_creds.collection) == 0 + + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + + message1 = self._record("mystream", "text data 1", 1) + message2 = self._record("mystream", "text data 2", 2) + + outcome = list(DestinationAstra().write(self.config, catalog, [message1, message2])) + assert astra_client.count_documents(db_creds.collection) == 2 + diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json b/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json new file mode 100644 index 00000000000000..a94caed893cf90 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json @@ -0,0 +1,377 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/astra", + "connectionSpecification": { + "title": "Destination Config", + "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", + "type": "object", + "properties": { + "embedding": { + "title": "Embedding", + "description": "Embedding configuration", + "group": "embedding", + "type": "object", + "oneOf": [ + { + "title": "OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai", + "const": "openai", + "enum": ["openai"], + "type": "string" + }, + "openai_key": { + "title": "OpenAI API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["openai_key", "mode"], + "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "Cohere", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "cohere", + "const": "cohere", + "enum": ["cohere"], + "type": "string" + }, + "cohere_key": { + "title": "Cohere API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["cohere_key", "mode"], + "description": "Use the Cohere API to embed text." + }, + { + "title": "Fake", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "fake", + "const": "fake", + "enum": ["fake"], + "type": "string" + } + }, + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", + "required": ["mode"] + }, + { + "title": "Azure OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "azure_openai", + "const": "azure_openai", + "enum": ["azure_openai"], + "type": "string" + }, + "openai_key": { + "title": "Azure OpenAI API key", + "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "airbyte_secret": true, + "type": "string" + }, + "api_base": { + "title": "Resource base URL", + "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["https://your-resource-name.openai.azure.com"], + "type": "string" + }, + "deployment": { + "title": "Deployment", + "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["your-resource-name"], + "type": "string" + } + }, + "required": ["openai_key", "api_base", "deployment", "mode"], + "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "OpenAI-compatible", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai_compatible", + "const": "openai_compatible", + "enum": ["openai_compatible"], + "type": "string" + }, + "api_key": { + "title": "API key", + "default": "", + "airbyte_secret": true, + "type": "string" + }, + "base_url": { + "title": "Base URL", + "description": "The base URL for your OpenAI-compatible service", + "examples": ["https://your-service-name.com"], + "type": "string" + }, + "model_name": { + "title": "Model name", + "description": "The name of the model to use for embedding", + "default": "text-embedding-ada-002", + "examples": ["text-embedding-ada-002"], + "type": "string" + }, + "dimensions": { + "title": "Embedding dimensions", + "description": "The number of dimensions the embedding model is generating", + "examples": [1536, 384], + "type": "integer" + } + }, + "required": ["base_url", "dimensions", "mode"], + "description": "Use a service that's compatible with the OpenAI API to embed text." + } + ] + }, + "processing": { + "title": "ProcessingConfigModel", + "type": "object", + "properties": { + "chunk_size": { + "title": "Chunk size", + "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "maximum": 8191, + "minimum": 1, + "type": "integer" + }, + "chunk_overlap": { + "title": "Chunk overlap", + "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + "default": 0, + "type": "integer" + }, + "text_fields": { + "title": "Text fields to embed", + "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + "default": [], + "always_show": true, + "examples": ["text", "user.name", "users.*.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "metadata_fields": { + "title": "Fields to store as metadata", + "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + "default": [], + "always_show": true, + "examples": ["age", "user", "user.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "text_splitter": { + "title": "Text splitter", + "description": "Split text fields into chunks based on the specified method.", + "type": "object", + "oneOf": [ + { + "title": "By Separator", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "separator", + "const": "separator", + "enum": ["separator"], + "type": "string" + }, + "separators": { + "title": "Separators", + "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", + "default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""], + "type": "array", + "items": { + "type": "string" + } + }, + "keep_separator": { + "title": "Keep separator", + "description": "Whether to keep the separator in the resulting chunks", + "default": false, + "type": "boolean" + } + }, + "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", + "required": ["mode"] + }, + { + "title": "By Markdown header", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "markdown", + "const": "markdown", + "enum": ["markdown"], + "type": "string" + }, + "split_level": { + "title": "Split level", + "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + "default": 1, + "minimum": 1, + "maximum": 6, + "type": "integer" + } + }, + "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", + "required": ["mode"] + }, + { + "title": "By Programming Language", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "code", + "const": "code", + "enum": ["code"], + "type": "string" + }, + "language": { + "title": "Language", + "description": "Split code in suitable places based on the programming language", + "enum": [ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol" + ], + "type": "string" + } + }, + "required": ["language", "mode"], + "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." + } + ] + }, + "field_name_mappings": { + "title": "Field name mappings", + "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + "default": [], + "type": "array", + "items": { + "title": "FieldNameMappingConfigModel", + "type": "object", + "properties": { + "from_field": { + "title": "From field name", + "description": "The field name in the source", + "type": "string" + }, + "to_field": { + "title": "To field name", + "description": "The field name to use in the destination", + "type": "string" + } + }, + "required": ["from_field", "to_field"] + } + } + }, + "required": ["chunk_size"], + "group": "processing" + }, + "omit_raw_text": { + "title": "Do not store raw text", + "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", + "default": false, + "group": "advanced", + "type": "boolean" + }, + "indexing": { + "title": "Indexing", + "type": "object", + "properties": { + "astra_db_app_token": { + "title": "AstraDB Application Token", + "description": "AstraDB Application Token", + "airbyte_secret": true, + "type": "string" + }, + "astra_db_endpoint": { + "title": "AstraDB Endpoint", + "description": "AstraDB Endpoint", + "pattern": "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\.]*?\\.apps\\.astra\\.datastax\\.com", + "examples": [ + "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com" + ], + "type": "string" + }, + "astra_db_keyspace": { + "title": "AstraDB Keyspace", + "description": "Astra DB Keyspace", + "type": "string" + }, + "collection": { + "title": "AstraDB collection", + "description": "AstraDB collection", + "type": "string" + } + }, + "required": [ + "astra_db_app_token", + "astra_db_endpoint", + "astra_db_keyspace", + "collection" + ], + "description": "Astra DB gives developers the APIs, real-time data and ecosystem integrations to put accurate RAG and Gen AI apps with fewer hallucinations in production.", + "group": "indexing" + } + }, + "required": ["embedding", "processing", "indexing"], + "groups": [ + { + "id": "processing", + "title": "Processing" + }, + { + "id": "embedding", + "title": "Embedding" + }, + { + "id": "indexing", + "title": "Indexing" + }, + { + "id": "advanced", + "title": "Advanced" + } + ] + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"] +} diff --git a/airbyte-integrations/connectors/destination-astra/main.py b/airbyte-integrations/connectors/destination-astra/main.py new file mode 100644 index 00000000000000..53b96b2b39ecba --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_astra import DestinationAstra + +if __name__ == "__main__": + DestinationAstra().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-astra/metadata.yaml b/airbyte-integrations/connectors/destination-astra/metadata.yaml new file mode 100644 index 00000000000000..c9c245c358d3f0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/metadata.yaml @@ -0,0 +1,30 @@ +data: + allowedHosts: + hosts: + - "*.apps.astra.datastax.com" + registries: + oss: + enabled: true + cloud: + enabled: true + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # Please use the full address with sha256 hash to guarantee build reproducibility. + # https://hub.docker.com/r/airbyte/python-connector-base + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 + connectorSubtype: database + connectorType: destination + definitionId: 042ce96f-1158-4662-9543-e2ff015be97a + dockerImageTag: 0.1.0 + dockerRepository: airbyte/destination-astra + githubIssueLabel: destination-astra + icon: astra.svg + license: MIT + name: Astra + releaseDate: 2024-01-10 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/destinations/astra + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-astra/requirements.txt b/airbyte-integrations/connectors/destination-astra/requirements.txt new file mode 100644 index 00000000000000..d6e1198b1ab1f5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-astra/setup.py b/airbyte-integrations/connectors/destination-astra/setup.py new file mode 100644 index 00000000000000..8bd1a185b52e7d --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0"] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_astra", + description="Destination implementation for Astra.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py new file mode 100644 index 00000000000000..f7d1400df70940 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, Mock, patch + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConnectorSpecification, Status +from destination_astra.config import ConfigModel +from destination_astra.destination import DestinationAstra + + +class TestDestinationAstra(unittest.TestCase): + def setUp(self): + self.config = { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "astra_db_app_token": "mytoken", + "astra_db_endpoint": "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com", + "astra_db_keyspace": "mykeyspace", + "collection": "mycollection", + }, + } + self.config_model = ConfigModel.parse_obj(self.config) + self.logger = AirbyteLogger() + + @patch("destination_astra.destination.AstraIndexer") + @patch("destination_astra.destination.create_from_config") + def test_check(self, MockedEmbedder, MockedAstraIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedAstraIndexer.return_value = mock_indexer + + mock_embedder.check.return_value = None + mock_indexer.check.return_value = None + + destination = DestinationAstra() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.SUCCEEDED) + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_astra.destination.AstraIndexer") + @patch("destination_astra.destination.create_from_config") + def test_check_with_errors(self, MockedEmbedder, MockedAstraIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedAstraIndexer.return_value = mock_indexer + + embedder_error_message = "Embedder Error" + indexer_error_message = "Indexer Error" + + mock_embedder.check.return_value = embedder_error_message + mock_indexer.check.return_value = indexer_error_message + + destination = DestinationAstra() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.FAILED) + self.assertEqual(result.message, f"{embedder_error_message}\n{indexer_error_message}") + + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_astra.destination.Writer") + @patch("destination_astra.destination.AstraIndexer") + @patch("destination_astra.destination.create_from_config") + def test_write(self, MockedEmbedder, MockedAstraIndexer, MockedWriter): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + mock_writer = Mock() + + MockedAstraIndexer.return_value = mock_indexer + MockedWriter.return_value = mock_writer + + mock_writer.write.return_value = [] + + configured_catalog = MagicMock() + input_messages = [] + + destination = DestinationAstra() + list(destination.write(self.config, configured_catalog, input_messages)) + + MockedWriter.assert_called_once_with(self.config_model.processing, mock_indexer, mock_embedder, batch_size=100, omit_raw_text=False) + mock_writer.write.assert_called_once_with(configured_catalog, input_messages) + + def test_spec(self): + destination = DestinationAstra() + result = destination.spec() + + self.assertIsInstance(result, ConnectorSpecification) diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py new file mode 100644 index 00000000000000..ebb1d41e230a01 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py @@ -0,0 +1,171 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from unittest.mock import ANY, MagicMock, Mock, patch + +import pytest +import urllib3 +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from destination_astra.config import AstraIndexingModel +from destination_astra.indexer import AstraIndexer + + +def create_astra_indexer(): + config = AstraIndexingModel( + astra_db_app_token="mytoken", + astra_db_endpoint="https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com", + astra_db_keyspace="mykeyspace", + collection="mycollection", + ) + indexer = AstraIndexer(config, 3) + + indexer.client.delete_documents = MagicMock() + indexer.client.insert_documents = MagicMock() + indexer.client.find_documents = MagicMock() + + return indexer + + +def create_index_description(collection_name, dimensions): + return {"name": collection_name, "options": {"vector": {"dimension": dimensions, "metric": "cosine"}}} + + +def test_astra_index_upsert_and_delete(): + indexer = create_astra_indexer() + indexer.index( + [ + Mock(page_content="test", metadata={"_ab_stream": "abc"}, embedding=[1, 2, 3]), + Mock(page_content="test2", metadata={"_ab_stream": "abc"}, embedding=[4, 5, 6]), + ], + "ns1", + "some_stream", + ) + indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") + indexer.client.delete_documents.assert_called_with( + collection_name="mycollection", filter={"_ab_record_id": {"$in": ["delete_id1", "delete_id2"]}} + ) + indexer.client.insert_documents.assert_called_with( + collection_name="mycollection", + documents=[ + {"_id": ANY, "$vector": [1, 2, 3], "_ab_stream": "abc", "text": "test"}, + {"_id": ANY, "$vector": [4, 5, 6], "_ab_stream": "abc", "text": "test2"}, + ], + ) + + +def test_astra_index_empty_batch(): + indexer = create_astra_indexer() + indexer.index([], "ns1", "some_stream") + indexer.client.delete_documents.assert_not_called() + indexer.client.insert_documents.assert_not_called() + + +def test_astra_index_upsert_batching(): + indexer = create_astra_indexer() + indexer.index( + [Mock(page_content=f"test {i}", metadata={"_ab_stream": "abc"}, embedding=[i, i, i]) for i in range(50)], + "ns1", + "some_stream", + ) + assert indexer.client.insert_documents.call_count == 3 + for i in range(20): + assert indexer.client.insert_documents.call_args_list[0].kwargs.get("documents")[i] == { + "_id": ANY, + "$vector": [i, i, i], + "_ab_stream": "abc", + "text": f"test {i}", + } + for i in range(20, 40): + assert indexer.client.insert_documents.call_args_list[1].kwargs.get("documents")[i - 20] == { + "_id": ANY, + "$vector": [i, i, i], + "_ab_stream": "abc", + "text": f"test {i}", + } + for i in range(40, 50): + assert indexer.client.insert_documents.call_args_list[2].kwargs.get("documents")[i - 40] == { + "_id": ANY, + "$vector": [i, i, i], + "_ab_stream": "abc", + "text": f"test {i}", + } + + +def generate_catalog(): + return ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + "namespace": "ns1", + }, + "primary_key": [["_id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + }, + { + "stream": { + "name": "example_stream2", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + "namespace": "ns2", + }, + "primary_key": [["_id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + ] + } + ) + + +def test_astra_pre_sync(): + indexer = create_astra_indexer() + indexer.client.find_collection = MagicMock(collection_name="") + indexer.client.find_collection.return_value = True + + indexer.pre_sync(generate_catalog()) + indexer.client.delete_documents.assert_called_with(collection_name="mycollection", filter={"_ab_stream": "ns2_example_stream2"}) + + +@pytest.mark.parametrize( + "collection_name,describe_throws,reported_dimensions,check_succeeds,error_message", + [ + ("mycollection", None, 3, True, None), + ("other_collection", None, 3, False, "mycollection collection does not exist."), + ( + ["mycollection"], + urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.")), + 3, + False, + "Failed to resolve environment", + ), + ("mycollection", None, 4, False, "Make sure embedding and indexing configurations match."), + ("mycollection", Exception("describe failed"), 3, False, "describe failed"), + ("mycollection", Exception("describe failed"), 4, False, "describe failed"), + ], +) +def test_astra_check(collection_name, describe_throws, reported_dimensions, check_succeeds, error_message): + indexer = create_astra_indexer() + + indexer.client.create_collection = MagicMock() + indexer.client.find_collections = MagicMock() + indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + + if describe_throws: + indexer.client.find_collections.side_effect = describe_throws + else: + indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + + result = indexer.check() + if check_succeeds: + assert result is None + else: + assert error_message in result diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/unit_test.py new file mode 100644 index 00000000000000..219ae0142c724f --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/unit_test.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +def test_example_method(): + assert True diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 9d4c49a4163df0..6b023138d6aa2d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.12.0' + cdkVersionRequired = '0.15.1' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 3ad145a95972b0..3d1dc837db2857 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.3.31 + dockerImageTag: 2.4.2 dockerRepository: airbyte/destination-bigquery documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery githubIssueLabel: destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index edec86e609917e..463439d02ae733 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -457,6 +457,11 @@ private TyperDeduper buildTyperDeduper(final BigQuerySqlGenerator sqlGenerator, } + @Override + public boolean isV2Destination() { + return true; + } + public static void main(final String[] args) throws Exception { AirbyteExceptionHandler.addThrowableForDeinterpolation(BigQueryException.class); final Destination destination = new BigQueryDestination(); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index a6f4f9a8c0cf54..4e79ae4fb23fdd 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -114,7 +114,7 @@ public void createStageIfNotExists(final String datasetId, final String stream) public String uploadRecordsToStage(final String datasetId, final String stream, final SerializableBuffer writer) { final String objectPath = getStagingFullPath(datasetId, stream); LOGGER.info("Uploading records to staging for stream {} (dataset {}): {}", stream, datasetId, objectPath); - return gcsStorageOperations.uploadRecordsToBucket(writer, datasetId, getStagingRootPath(datasetId, stream), objectPath); + return gcsStorageOperations.uploadRecordsToBucket(writer, datasetId, objectPath); } /** diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json index 0b366183eb523a..b291203568e8e6 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json @@ -211,7 +211,7 @@ }, "use_1s1t_format": { "type": "boolean", - "description": "(Early Access) Use Destinations V2.", + "description": "(Early Access) Use Destinations V2.", "title": "Use Destinations V2 (Early Access)", "order": 9 }, diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index da4dcd05868d7b..0bc39653b8f08c 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.13.1' + cdkVersionRequired = '0.14.0' features = [ 'db-sources', // required for tests 'db-destinations', diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json index cf067a7338d92f..86900109c06c57 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json @@ -218,7 +218,7 @@ }, "use_1s1t_format": { "type": "boolean", - "description": "(Early Access) Use Destinations V2.", + "description": "(Early Access) Use Destinations V2.", "title": "Use Destinations V2 (Early Access)", "order": 9 }, diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index aa75211ebf3e72..359a816d4fe79d 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.13.0' + cdkVersionRequired = '0.15.1' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index b32972ffd65e14..7c9407b8aa95ca 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,22 +5,25 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 0.8.0 + dockerImageTag: 2.1.3 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift icon: redshift.svg license: MIT name: Redshift - normalizationConfig: - normalizationIntegrationType: redshift - normalizationRepository: airbyte/normalization-redshift - normalizationTag: 0.4.3 registries: cloud: enabled: true oss: enabled: true + releases: + breakingChanges: + 2.0.0: + message: > + This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. + upgradeDeadline: "2024-03-15" releaseStage: beta resourceRequirements: jobSpecific: diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java index d10cabd45730b9..5b48a24ca115be 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java @@ -69,6 +69,11 @@ public ConnectorSpecification spec() throws Exception { return originalSpec; } + @Override + public boolean isV2Destination() { + return true; + } + public static void main(final String[] args) throws Exception { final Destination destination = new RedshiftDestination(); LOGGER.info("starting destination: {}", RedshiftDestination.class); diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index 82af7555922b5c..6073186a51f4da 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -42,7 +42,6 @@ import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; import io.airbyte.integrations.base.destination.typing_deduping.NoOpTyperDeduperWithV1V2Migrations; -import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; import io.airbyte.integrations.base.destination.typing_deduping.NoopV2TableMigrator; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; @@ -206,25 +205,6 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN of streams {} this will create more buffers than necessary, leading to nonexistent gains """, FileBuffer.SOFT_CAP_CONCURRENT_STREAM_IN_BUFFER, catalog.getStreams().size()); } - // Short circuit old way of running things during transition. - if (!TypingAndDedupingFlag.isDestinationV2()) { - return new StagingConsumerFactory().createAsync( - outputRecordCollector, - getDatabase(getDataSource(config)), - new RedshiftS3StagingSqlOperations(getNamingResolver(), s3Config.getS3Client(), s3Config, encryptionConfig), - getNamingResolver(), - config, - catalog, - isPurgeStagingData(s3Options), - new TypeAndDedupeOperationValve(), - new NoopTyperDeduper(), - // The parsedcatalog is only used in v2 mode, so just pass null for now - null, - // Overwriting null namespace with null is perfectly safe - null, - // still using v1 table format - false); - } final String defaultNamespace = config.get("schema").asText(); for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java index 9cf38f7ce4af1d..44dee9e63591e6 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java @@ -71,7 +71,12 @@ public String getStagingPath(final UUID connectionId, } @Override - public void createStageIfNotExists() throws Exception { + public String getStageName(String namespace, String streamName) { + return "garbage-unused"; + } + + @Override + public void createStageIfNotExists(final JdbcDatabase database, final String stageName) throws Exception { s3StorageOperations.createBucketIfNotExists(); } @@ -79,6 +84,7 @@ public void createStageIfNotExists() throws Exception { public String uploadRecordsToStage(final JdbcDatabase database, final SerializableBuffer recordsData, final String schemaName, + final String stageName, final String stagingPath) throws Exception { return s3StorageOperations.uploadRecordsToBucket(recordsData, schemaName, stagingPath); @@ -92,6 +98,7 @@ private String putManifest(final String manifestContents, final String stagingPa @Override public void copyIntoTableFromStage(final JdbcDatabase database, + final String stageName, final String stagingPath, final List stagedFiles, final String tableName, @@ -160,8 +167,9 @@ private static String getManifestPath(final String s3BucketName, final String s3 } @Override - public void dropStageIfExists(final JdbcDatabase database, final String stageName) throws Exception { - s3StorageOperations.dropBucketObject(stageName); + public void dropStageIfExists(final JdbcDatabase database, final String stageName, final String stagingPath) throws Exception { + // stageName is unused here but used in Snowflake. This interface needs to be fixed. + s3StorageOperations.dropBucketObject(stagingPath); } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java index 7b85a0d92706a0..a25a4b53846f5a 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java @@ -31,6 +31,7 @@ import org.jooq.InsertValuesStep4; import org.jooq.Record; import org.jooq.SQLDialect; +import org.jooq.conf.ParamType; import org.jooq.impl.DefaultDataType; import org.jooq.impl.SQLDataType; import org.slf4j.Logger; @@ -133,11 +134,17 @@ protected void insertRecordsInternalV2(final JdbcDatabase database, for (final PartialAirbyteMessage record : batch) { insert = insert.values( val(UUID.randomUUID().toString()), - function("JSON_PARSE", String.class, val(record.getSerialized())), + function("JSON_PARSE", String.class, val(escapeStringLiteral(record.getSerialized()))), val(Instant.ofEpochMilli(record.getRecord().getEmittedAt()).atOffset(ZoneOffset.UTC)), val((OffsetDateTime) null)); } - insert.execute(); + // Intentionally don't use insert.execute(). + // jooq will try to use a parameterized query if there are not too many query params. + // This means that for small queries, we would need to not-escape backslashes, + // but for large queries we _would_ need to escape them. + // Instead, force jooq to always inline params. + // This allows us to always escape backslashes. + connection.createStatement().execute(insert.getSQL(ParamType.INLINED)); LOGGER.info("Executed batch size: {}, {}, {}", batch.size(), schemaName, tableName); } }); @@ -147,4 +154,15 @@ protected void insertRecordsInternalV2(final JdbcDatabase database, } } + public static String escapeStringLiteral(final String str) { + if (str == null) { + return null; + } else { + // jooq handles most things + // but we need to manually escape backslashes because postgres and redshift have + // different backslash handling. + return str.replace("\\", "\\\\"); + } + } + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java index 3fe5e6ecf32d64..a9b0ec0330e030 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java @@ -49,6 +49,16 @@ public void execute(final Sql sql) throws Exception { } } + /** + * Issuing a select 1 limit 1 query can be expensive, so relying on SVV_TABLE_INFO system table. + * EXPLAIN of the select 1 from table limit 1 query: (seq scan and then limit is applied, read from + * bottom to top) XN Lim it (co st=0. 0 .0.01 rows=1 width=0) -> XN Seq Scan on _airbyte_raw_ users + * (cost=0.00..1000.00 rows=100000 width=0) + * + * @param id + * @return + * @throws Exception + */ @Override public boolean isFinalTableEmpty(final StreamId id) throws Exception { // Redshift doesn't have an information_schema.tables table, so we have to use SVV_TABLE_INFO. diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index 28a274642bbf76..55cb60c52a62fb 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -249,34 +249,27 @@ } ] }, - "use_1s1t_format": { - "type": "boolean", - "description": "(Early Access) Use Destinations V2.", - "title": "Use Destinations V2 (Early Access)", - "order": 9, - "group": "connection" - }, "raw_data_schema": { "type": "string", - "description": "(Early Access) The schema to write raw tables into", - "title": "Destinations V2 Raw Table Schema (Early Access)", - "order": 10, + "description": "The schema to write raw tables into", + "title": "Destinations V2 Raw Table Schema", + "order": 9, "group": "connection" }, "enable_incremental_final_table_updates": { "type": "boolean", "default": false, "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", - "title": "Enable Loading Data Incrementally to Final Tables (Early Access)", - "order": 11, + "title": "Enable Loading Data Incrementally to Final Tables", + "order": 10, "group": "connection" }, "disable_type_dedupe": { "type": "boolean", "default": false, "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", - "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions) (Early Access)", - "order": 12, + "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", + "order": 11, "group": "connection" } }, diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java index 6ca0a17fce3cee..59e9130af79fd6 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java @@ -9,10 +9,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; /** * Integration test testing the {@link RedshiftInsertDestination}. */ +@Disabled public class RedshiftInsertDestinationAcceptanceTest extends RedshiftDestinationAcceptanceTest { public JsonNode getStaticConfig() throws IOException { diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java index fc054be5123266..0732e604145470 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java @@ -8,11 +8,13 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; /** * Integration test testing {@link RedshiftStagingS3Destination}. The default Redshift integration * test credentials contain S3 credentials - this automatically causes COPY to be selected. */ +@Disabled public class RedshiftS3StagingInsertDestinationAcceptanceTest extends RedshiftDestinationAcceptanceTest { public JsonNode getStaticConfig() { diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java index 1d80069433daa4..4c2da0a04ce77b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java @@ -10,11 +10,13 @@ import io.airbyte.commons.json.Jsons; import java.io.IOException; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; /* * SshKeyRedshiftInsertDestinationAcceptanceTest runs basic Redshift Destination Tests using the SQL * Insert mechanism for upload of data and "key" authentication for the SSH bastion configuration. */ +@Disabled public class SshKeyRedshiftInsertDestinationAcceptanceTest extends SshRedshiftDestinationBaseAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java index 6f423e5e43d127..72c992fdb18358 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java @@ -10,12 +10,14 @@ import io.airbyte.commons.json.Jsons; import java.io.IOException; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; /* * SshPasswordRedshiftStagingDestinationAcceptanceTest runs basic Redshift Destination Tests using * the S3 Staging mechanism for upload of data and "password" authentication for the SSH bastion * configuration. */ +@Disabled public class SshPasswordRedshiftStagingDestinationAcceptanceTest extends SshRedshiftDestinationBaseAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java index 4ac040794f2610..1ddca62a2bcda4 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLTime; +import static io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations.escapeStringLiteral; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -205,14 +206,4 @@ public void testCreateTableIncremental() throws Exception { // TODO assert on table clustering, etc. } - private static String escapeStringLiteral(final String str) { - if (str == null) { - return null; - } else { - // jooq handles most things - // but we need to manually escape backslashes for some reason - return str.replace("\\", "\\\\"); - } - } - } diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index 4ce8698d75d8f7..3f3842a001e606 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.12.0' + cdkVersionRequired = '0.15.1' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index b5d7311d68df23..9cacaccb4ab585 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.4.22 + dockerImageTag: 3.5.4 dockerRepository: airbyte/destination-snowflake documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake githubIssueLabel: destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index 217ed51abb78ac..012f0846e41c1d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -40,4 +40,9 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN return new SnowflakeInternalStagingDestination(airbyteEnvironment).getSerializedMessageConsumer(config, catalog, outputRecordCollector); } + @Override + public boolean isV2Destination() { + return true; + } + } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index f0f2732518ad6e..af4f3961b4afa2 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -99,7 +99,7 @@ private static void attemptStageOperations(final String outputSchema, sqlOperations.attemptWriteToStage(outputSchema, stageName, database); } finally { // drop created tmp stage - sqlOperations.dropStageIfExists(database, stageName); + sqlOperations.dropStageIfExists(database, stageName, null); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java index d789fe7e5c26a7..c969b86e31a719 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java @@ -202,7 +202,7 @@ protected String getCopyQuery(final String stageName, } @Override - public void dropStageIfExists(final JdbcDatabase database, final String stageName) throws Exception { + public void dropStageIfExists(final JdbcDatabase database, final String stageName, final String stagingPath) throws Exception { try { final String query = getDropQuery(stageName); LOGGER.debug("Executing query: {}", query); @@ -222,17 +222,6 @@ protected String getDropQuery(final String stageName) { return String.format(DROP_STAGE_QUERY, stageName); } - @Override - public void cleanUpStage(final JdbcDatabase database, final String stageName, final List stagedFiles) throws Exception { - try { - final String query = getRemoveQuery(stageName); - LOGGER.debug("Executing query: {}", query); - database.execute(query); - } catch (final SQLException e) { - throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); - } - } - /** * Creates a SQL query used to remove staging files that were just staged See * https://docs.snowflake.com/en/sql-reference/sql/remove.html for more context diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java index 6be94ea5032fe6..be9ff16282f73f 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java @@ -109,8 +109,9 @@ public void insertRecordsInternal(final JdbcDatabase database, @Override protected void insertRecordsInternalV2(final JdbcDatabase jdbcDatabase, final List list, final String s, final String s1) throws Exception { - // Snowflake doesn't have standard inserts... so we probably never want to do this - throw new UnsupportedOperationException("Snowflake does not use the native JDBC DV2 interface"); + // Snowflake doesn't have standard inserts... so we don't do this at real runtime. + // Intentionally do nothing. This method is called from the `check` method. + // It probably shouldn't be, but this is the easiest path to getting this working. } protected String generateFilesList(final List files) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java index 06374e1fe6135d..5ab8d85e6c575d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java @@ -49,7 +49,6 @@ class SnowflakeSqlOperationsThrowConfigExceptionTest { private static Executable createStageIfNotExists; private static Executable dropStageIfExists; - private static Executable cleanUpStage; private static Executable copyIntoTableFromStage; private static Executable createSchemaIfNotExists; @@ -65,8 +64,7 @@ public static void setup() { snowflakeSqlOperations = new SnowflakeSqlOperations(); createStageIfNotExists = () -> snowflakeStagingSqlOperations.createStageIfNotExists(dbForExecuteQuery, STAGE_NAME); - dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME); - cleanUpStage = () -> snowflakeStagingSqlOperations.cleanUpStage(dbForExecuteQuery, STAGE_NAME, FILE_PATH); + dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME, null); copyIntoTableFromStage = () -> snowflakeStagingSqlOperations.copyIntoTableFromStage(dbForExecuteQuery, STAGE_NAME, STAGE_PATH, FILE_PATH, TABLE_NAME, SCHEMA_NAME); @@ -84,9 +82,6 @@ private static Stream testArgumentsForDbExecute() { Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, dropStageIfExists), Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, dropStageIfExists), Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, dropStageIfExists), - Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, cleanUpStage), - Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, cleanUpStage), - Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, cleanUpStage), Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, copyIntoTableFromStage), Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, copyIntoTableFromStage), Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, copyIntoTableFromStage), diff --git a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml index 29aeefaf0831e2..ebd5ba581c6e60 100644 --- a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml +++ b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 7b7d7a0d-954c-45a0-bcfc-39a634b97736 - dockerImageTag: 0.2.14 + dockerImageTag: 0.2.15 dockerRepository: airbyte/destination-weaviate documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate githubIssueLabel: destination-weaviate diff --git a/airbyte-integrations/connectors/destination-weaviate/setup.py b/airbyte-integrations/connectors/destination-weaviate/setup.py index ca4aa59495ddc7..0a49aa856b74ab 100644 --- a/airbyte-integrations/connectors/destination-weaviate/setup.py +++ b/airbyte-integrations/connectors/destination-weaviate/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0", "weaviate-client==3.25.2"] -TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker"] +TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker==2.0.1"] setup( name="destination_weaviate", diff --git a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl index 9b65df5c424e5e..f2cc5cdef9d9f0 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl @@ -24,7 +24,7 @@ {"stream": "pull_request_stats", "data": {"node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "id": 721435506, "number": 5, "updated_at": "2023-11-16T14:38:58Z", "changed_files": 5, "deletions": 0, "additions": 5, "merged": false, "mergeable": "MERGEABLE", "can_be_rebased": false, "maintainer_can_modify": false, "merge_state_status": "BLOCKED", "comments": 0, "commits": 5, "review_comments": 0, "merged_by": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1700557306144} {"stream": "pull_requests", "data": {"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5", "id": 721435506, "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "html_url": "https://github.com/airbytehq/integration-test/pull/5", "diff_url": "https://github.com/airbytehq/integration-test/pull/5.diff", "patch_url": "https://github.com/airbytehq/integration-test/pull/5.patch", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5", "number": 5, "state": "closed", "locked": false, "title": "New PR from feature/branch_4", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "body": null, "created_at": "2021-08-27T15:43:40Z", "updated_at": "2023-11-16T14:38:58Z", "closed_at": "2023-11-16T14:38:58Z", "merged_at": null, "merge_commit_sha": "191309e3da8b36705156348ae73f4dca836533f9", "assignee": null, "assignees": [], "requested_reviewers": [], "requested_teams": [], "labels": [{"id": 3295756566, "node_id": "MDU6TGFiZWwzMjk1NzU2NTY2", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/bug", "name": "bug", "color": "d73a4a", "default": true, "description": "Something isn't working"}, {"id": 3300346197, "node_id": "MDU6TGFiZWwzMzAwMzQ2MTk3", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/critical", "name": "critical", "color": "ededed", "default": false, "description": null}], "milestone": null, "draft": false, "commits_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits", "review_comments_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments", "review_comment_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "head": {"label": "airbytehq:feature/branch_4", "ref": "feature/branch_4", "sha": "31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo_id": 400052213}, "base": {"label": "airbytehq:master", "ref": "master", "sha": "978753aeb56f7b49872279d1b491411a6235aa90", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments", "created_at": "2021-08-26T05:32:43Z", "updated_at": "2023-11-16T14:48:53Z", "pushed_at": "2023-05-03T16:40:56Z", "git_url": "git://github.com/airbytehq/integration-test.git", "ssh_url": "git@github.com:airbytehq/integration-test.git", "clone_url": "https://github.com/airbytehq/integration-test.git", "svn_url": "https://github.com/airbytehq/integration-test", "homepage": null, "size": 11, "stargazers_count": 4, "watchers_count": 4, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 2, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 6, "license": null, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [], "visibility": "public", "forks": 2, "open_issues": 6, "watchers": 4, "default_branch": "master"}, "repo_id": null}, "_links": {"self": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5"}, "html": {"href": "https://github.com/airbytehq/integration-test/pull/5"}, "issue": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5"}, "comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments"}, "review_comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments"}, "review_comment": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}"}, "commits": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits"}, "statuses": {"href": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083"}}, "author_association": "CONTRIBUTOR", "auto_merge": null, "active_lock_reason": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1700585060024} {"stream":"releases","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586","assets_url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586/assets","upload_url":"https://uploads.github.com/repos/airbytehq/integration-test/releases/48581586/assets{?name,label}","html_url":"https://github.com/airbytehq/integration-test/releases/tag/dev-0.9","id":48581586,"author":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"node_id":"MDc6UmVsZWFzZTQ4NTgxNTg2","tag_name":"dev-0.9","target_commitish":"master","name":"9 global release","draft":false,"prerelease":false,"created_at":"2021-08-27T07:03:09Z","published_at":"2021-08-27T15:43:53Z","assets":[],"tarball_url":"https://api.github.com/repos/airbytehq/integration-test/tarball/dev-0.9","zipball_url":"https://api.github.com/repos/airbytehq/integration-test/zipball/dev-0.9","body":"","repository":"airbytehq/integration-test"},"emitted_at":1677668760424} -{"stream": "repositories", "data": {"id": 283046497, "node_id": "MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=", "name": "airbyte", "full_name": "airbytehq/airbyte", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/airbyte", "description": "Data integration platform for ELT pipelines from APIs, databases & files to warehouses & lakes.", "fork": false, "url": "https://api.github.com/repos/airbytehq/airbyte", "forks_url": "https://api.github.com/repos/airbytehq/airbyte/forks", "keys_url": "https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/airbyte/teams", "hooks_url": "https://api.github.com/repos/airbytehq/airbyte/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/airbyte/events", "assignees_url": "https://api.github.com/repos/airbytehq/airbyte/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/airbyte/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/airbyte/tags", "blobs_url": "https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/airbyte/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/airbyte/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/airbyte/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/airbyte/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/airbyte/subscription", "commits_url": "https://api.github.com/repos/airbytehq/airbyte/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/airbyte/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/airbyte/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/airbyte/merges", "archive_url": "https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/airbyte/downloads", "issues_url": "https://api.github.com/repos/airbytehq/airbyte/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/airbyte/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/airbyte/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/airbyte/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/airbyte/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/airbyte/deployments", "created_at": "2020-07-27T23:55:54Z", "updated_at": "2023-11-21T14:55:05Z", "pushed_at": "2023-11-21T16:55:37Z", "git_url": "git://github.com/airbytehq/airbyte.git", "ssh_url": "git@github.com:airbytehq/airbyte.git", "clone_url": "https://github.com/airbytehq/airbyte.git", "svn_url": "https://github.com/airbytehq/airbyte", "homepage": "https://airbyte.com", "size": 455477, "stargazers_count": 12328, "watchers_count": 12328, "language": "Python", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": false, "has_pages": false, "has_discussions": true, "forks_count": 3226, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 5053, "license": {"key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA="}, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": ["airbyte", "bigquery", "change-data-capture", "data", "data-analysis", "data-collection", "data-engineering", "data-ingestion", "data-integration", "elt", "etl", "java", "pipeline", "python", "redshift", "snowflake"], "visibility": "public", "forks": 3226, "open_issues": 5053, "watchers": 12328, "default_branch": "master", "permissions": {"admin": true, "maintain": true, "push": true, "triage": true, "pull": true}, "security_and_analysis": {"secret_scanning": {"status": "disabled"}, "secret_scanning_push_protection": {"status": "disabled"}, "dependabot_security_updates": {"status": "enabled"}, "secret_scanning_validity_checks": {"status": "disabled"}}, "organization": "airbytehq"}, "emitted_at": 1700585836592} +{"stream":"repositories","data":{"id":283046497,"node_id":"MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=","name":"airbyte","full_name":"airbytehq/airbyte","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/airbyte","description":"The leading data integration platform for ETL / ELT data pipelines from APIs, databases & files to data warehouses, data lakes & data lakehouses. Both self-hosted and Cloud-hosted.","fork":false,"url":"https://api.github.com/repos/airbytehq/airbyte","forks_url":"https://api.github.com/repos/airbytehq/airbyte/forks","keys_url":"https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/airbyte/teams","hooks_url":"https://api.github.com/repos/airbytehq/airbyte/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/airbyte/events","assignees_url":"https://api.github.com/repos/airbytehq/airbyte/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/airbyte/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/airbyte/tags","blobs_url":"https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/airbyte/languages","stargazers_url":"https://api.github.com/repos/airbytehq/airbyte/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/airbyte/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/airbyte/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/airbyte/subscription","commits_url":"https://api.github.com/repos/airbytehq/airbyte/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/airbyte/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/airbyte/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/airbyte/merges","archive_url":"https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/airbyte/downloads","issues_url":"https://api.github.com/repos/airbytehq/airbyte/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/airbyte/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/airbyte/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/airbyte/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/airbyte/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/airbyte/deployments","created_at":"2020-07-27T23:55:54Z","updated_at":"2024-01-26T13:38:04Z","pushed_at":"2024-01-26T13:46:31Z","git_url":"git://github.com/airbytehq/airbyte.git","ssh_url":"git@github.com:airbytehq/airbyte.git","clone_url":"https://github.com/airbytehq/airbyte.git","svn_url":"https://github.com/airbytehq/airbyte","homepage":"https://airbyte.com","size":486685,"stargazers_count":12924,"watchers_count":12924,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"has_discussions":true,"forks_count":3381,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":5107,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"allow_forking":true,"is_template":false,"web_commit_signoff_required":false,"topics":["bigquery","change-data-capture","data","data-analysis","data-collection","data-engineering","data-integration","data-pipeline","elt","etl","java","mssql","mysql","pipeline","postgresql","python","redshift","s3","self-hosted","snowflake"],"visibility":"public","forks":3381,"open_issues":5107,"watchers":12924,"default_branch":"master","permissions":{"admin":true,"maintain":true,"push":true,"triage":true,"pull":true},"security_and_analysis":{"secret_scanning":{"status":"disabled"},"secret_scanning_push_protection":{"status":"disabled"},"dependabot_security_updates":{"status":"enabled"},"secret_scanning_validity_checks":{"status":"disabled"}},"organization":"airbytehq"},"emitted_at":1706276794871} {"stream":"review_comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726","pull_request_review_id":742633128,"id":699253726,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDY5OTI1MzcyNg==","diff_hunk":"@@ -0,0 +1 @@\n+text_for_file_","path":"github_sources/file_1.txt","commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","original_commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"body":"Good point","created_at":"2021-08-31T12:01:15Z","updated_at":"2021-08-31T12:01:15Z","html_url":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726","pull_request_url":"https://api.github.com/repos/airbytehq/integration-test/pulls/4","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726"},"html":{"href":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726"},"pull_request":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/4"}},"reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726/reactions","total_count":1,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":1,"rocket":0,"eyes":0},"start_line":null,"original_start_line":null,"start_side":null,"line":1,"original_line":1,"side":"RIGHT","original_position":1,"position":1,"subject_type":"line","repository":"airbytehq/integration-test"},"emitted_at":1695375624151} {"stream":"reviews","data":{"node_id":"MDE3OlB1bGxSZXF1ZXN0UmV2aWV3NzQwNjU5Nzk4","id":740659798,"body":"Review commit for branch feature/branch_4","state":"COMMENTED","html_url":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798","author_association":"CONTRIBUTOR","submitted_at":"2021-08-27T15:43:42Z","created_at":"2021-08-27T15:43:42Z","updated_at":"2021-08-27T15:43:42Z","user":{"node_id":"MDQ6VXNlcjc0MzkwMQ==","id":743901,"login":"gaart","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","html_url":"https://github.com/gaart","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","pull_request_url":"https://github.com/airbytehq/integration-test/pull/5","commit_id":"31a3e3f19fefce60fba6bfc69dd2b3fb5195a083","_links":{"html":{"href":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798"},"pull_request":{"href":"https://github.com/airbytehq/integration-test/pull/5"}}},"emitted_at":1677668764954} {"stream":"stargazers","data":{"starred_at":"2021-08-27T16:23:34Z","user":{"login":"VasylLazebnyk","id":68591643,"node_id":"MDQ6VXNlcjY4NTkxNjQz","avatar_url":"https://avatars.githubusercontent.com/u/68591643?v=4","gravatar_id":"","url":"https://api.github.com/users/VasylLazebnyk","html_url":"https://github.com/VasylLazebnyk","followers_url":"https://api.github.com/users/VasylLazebnyk/followers","following_url":"https://api.github.com/users/VasylLazebnyk/following{/other_user}","gists_url":"https://api.github.com/users/VasylLazebnyk/gists{/gist_id}","starred_url":"https://api.github.com/users/VasylLazebnyk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VasylLazebnyk/subscriptions","organizations_url":"https://api.github.com/users/VasylLazebnyk/orgs","repos_url":"https://api.github.com/users/VasylLazebnyk/repos","events_url":"https://api.github.com/users/VasylLazebnyk/events{/privacy}","received_events_url":"https://api.github.com/users/VasylLazebnyk/received_events","type":"User","site_admin":false},"repository":"airbytehq/integration-test","user_id":68591643},"emitted_at":1677668765231} diff --git a/airbyte-integrations/connectors/source-github/metadata.yaml b/airbyte-integrations/connectors/source-github/metadata.yaml index cece9f362bd402..ecb9c0fbbee994 100644 --- a/airbyte-integrations/connectors/source-github/metadata.yaml +++ b/airbyte-integrations/connectors/source-github/metadata.yaml @@ -6,11 +6,11 @@ data: hosts: - ${api_url} connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e - dockerImageTag: 1.5.5 + dockerImageTag: 1.5.6 dockerRepository: airbyte/source-github documentationUrl: https://docs.airbyte.com/integrations/sources/github githubIssueLabel: source-github diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index 04e02fbadf2101..cf440b6b0cfbf6 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -123,14 +123,7 @@ def get_access_token(config: Mapping[str, Any]): def _get_authenticator(self, config: Mapping[str, Any]): _, token = self.get_access_token(config) tokens = [t.strip() for t in token.split(constants.TOKEN_SEPARATOR)] - requests_per_hour = config.get("requests_per_hour") - if requests_per_hour: - return MultipleTokenAuthenticatorWithRateLimiter( - tokens=tokens, - auth_method="token", - requests_per_hour=requests_per_hour, - ) - return MultipleTokenAuthenticator(tokens=tokens, auth_method="token") + return MultipleTokenAuthenticatorWithRateLimiter(tokens=tokens) def _validate_and_transform_config(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: config = self._ensure_default_values(config) diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json index 8c24d76278e7d5..edfb6f9a6c3989 100644 --- a/airbyte-integrations/connectors/source-github/source_github/spec.json +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -130,13 +130,6 @@ "description": "List of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.", "order": 4, "pattern_descriptor": "org/repo/branch1 org/repo/branch2" - }, - "requests_per_hour": { - "type": "integer", - "title": "Max requests per hour", - "description": "The GitHub API allows for a maximum of 5000 requests per hour (15000 for Github Enterprise). You can specify a lower value to limit your use of the API quota.", - "minimum": 1, - "order": 5 } } }, diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 3f7d710d04e8e0..c8dc6216d06516 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -25,7 +25,7 @@ get_query_pull_requests, get_query_reviews, ) -from .utils import getter +from .utils import GitHubAPILimitException, getter class GithubStreamABC(HttpStream, ABC): @@ -38,6 +38,8 @@ class GithubStreamABC(HttpStream, ABC): stream_base_params = {} def __init__(self, api_url: str = "https://api.github.com", access_token_type: str = "", **kwargs): + if kwargs.get("authenticator"): + kwargs["authenticator"].max_time = self.max_time super().__init__(**kwargs) self.access_token_type = access_token_type @@ -126,16 +128,25 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: # we again could have 5000 per another hour. min_backoff_time = 60.0 - retry_after = response.headers.get("Retry-After") if retry_after is not None: - return max(float(retry_after), min_backoff_time) + backoff_time_in_seconds = max(float(retry_after), min_backoff_time) + return self.get_waiting_time(backoff_time_in_seconds) reset_time = response.headers.get("X-RateLimit-Reset") if reset_time: - return max(float(reset_time) - time.time(), min_backoff_time) + backoff_time_in_seconds = max(float(reset_time) - time.time(), min_backoff_time) + return self.get_waiting_time(backoff_time_in_seconds) + + def get_waiting_time(self, backoff_time_in_seconds): + if backoff_time_in_seconds < self.max_time: + return backoff_time_in_seconds + else: + self._session.auth.update_token() # New token will be used in next request + return 1 - def check_graphql_rate_limited(self, response_json) -> bool: + @staticmethod + def check_graphql_rate_limited(response_json: dict) -> bool: errors = response_json.get("errors") if errors: for error in errors: @@ -203,6 +214,8 @@ def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iter raise e self.logger.warning(error_msg) + except GitHubAPILimitException: + self.logger.warning("Limits for all provided tokens are reached, please try again later") class GithubStream(GithubStreamABC): diff --git a/airbyte-integrations/connectors/source-github/source_github/utils.py b/airbyte-integrations/connectors/source-github/source_github/utils.py index 285582d815beb9..7907c29b636a33 100644 --- a/airbyte-integrations/connectors/source-github/source_github/utils.py +++ b/airbyte-integrations/connectors/source-github/source_github/utils.py @@ -2,14 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging import time +from dataclasses import dataclass from itertools import cycle -from types import SimpleNamespace -from typing import List +from typing import Any, List, Mapping +import pendulum +import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator @@ -32,6 +34,18 @@ def read_full_refresh(stream_instance: Stream): yield record +class GitHubAPILimitException(Exception): + """General class for Rate Limits errors""" + + +@dataclass +class Token: + count_rest: int = 5000 + count_graphql: int = 5000 + reset_at_rest: pendulum.DateTime = pendulum.now() + reset_at_graphql: pendulum.DateTime = pendulum.now() + + class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator): """ Each token in the cycle is checked against the rate limiter. @@ -40,49 +54,99 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator): the first token becomes available again. """ - DURATION = 3600 # seconds + DURATION = pendulum.duration(seconds=3600) # Duration at which the current rate limit window resets - def __init__(self, tokens: List[str], requests_per_hour: int, auth_method: str = "Bearer", auth_header: str = "Authorization"): + def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"): self._auth_method = auth_method self._auth_header = auth_header - now = time.time() - self._requests_per_hour = requests_per_hour - self._tokens = {t: SimpleNamespace(count=self._requests_per_hour, update_at=now) for t in tokens} + self._tokens = {t: Token() for t in tokens} + self.check_all_tokens() self._tokens_iter = cycle(self._tokens) + self._active_token = next(self._tokens_iter) + self._max_time = 60 * 10 # 10 minutes as default @property def auth_header(self) -> str: return self._auth_header + def get_auth_header(self) -> Mapping[str, Any]: + """The header to set on outgoing HTTP requests""" + if self.auth_header: + return {self.auth_header: self.token} + return {} + + def __call__(self, request): + """Attach the HTTP headers required to authenticate on the HTTP request""" + while True: + current_token = self._tokens[self.current_active_token] + if "graphql" in request.path_url: + if self.process_token(current_token, "count_graphql", "reset_at_graphql"): + break + else: + if self.process_token(current_token, "count_rest", "reset_at_rest"): + break + + request.headers.update(self.get_auth_header()) + + return request + + @property + def current_active_token(self) -> str: + return self._active_token + + def update_token(self) -> None: + self._active_token = next(self._tokens_iter) + @property def token(self) -> str: - while True: - token = next(self._tokens_iter) - if self._check_token(token): - return f"{self._auth_method} {token}" - def _check_token(self, token: str): + token = self.current_active_token + return f"{self._auth_method} {token}" + + @property + def max_time(self) -> int: + return self._max_time + + @max_time.setter + def max_time(self, value: int) -> None: + self._max_time = value + + def _check_token_limits(self, token: str): """check that token is not limited""" - self._refill() - if self._sleep(): - self._refill() - if self._tokens[token].count > 0: - self._tokens[token].count -= 1 - return True + headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} + rate_limit_info = ( + requests.get( + "https://api.github.com/rate_limit", headers=headers, auth=TokenAuthenticator(token, auth_method=self._auth_method) + ) + .json() + .get("resources") + ) + token_info = self._tokens[token] + remaining_info_core = rate_limit_info.get("core") + token_info.count_rest, token_info.reset_at_rest = remaining_info_core.get("remaining"), pendulum.from_timestamp( + remaining_info_core.get("reset") + ) + + remaining_info_graphql = rate_limit_info.get("graphql") + token_info.count_graphql, token_info.reset_at_graphql = remaining_info_graphql.get("remaining"), pendulum.from_timestamp( + remaining_info_graphql.get("reset") + ) - def _refill(self): - """refill all needed tokens""" - now = time.time() - for token, ns in self._tokens.items(): - if now - ns.update_at >= self.DURATION: - ns.update_at = now - ns.count = self._requests_per_hour - - def _sleep(self): - """sleep only if all tokens is exhausted""" - now = time.time() - if sum([ns.count for ns in self._tokens.values()]) == 0: - sleep_time = self.DURATION - (now - min([ns.update_at for ns in self._tokens.values()])) - logging.warning("Sleeping for %.1f seconds to enforce the limit of %d requests per hour.", sleep_time, self._requests_per_hour) - time.sleep(sleep_time) + def check_all_tokens(self): + for token in self._tokens: + self._check_token_limits(token) + + def process_token(self, current_token, count_attr, reset_attr): + if getattr(current_token, count_attr) > 0: + setattr(current_token, count_attr, getattr(current_token, count_attr) - 1) return True + elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()): + min_time_to_wait = min((getattr(x, reset_attr) - pendulum.now()).in_seconds() for x in self._tokens.values()) + if min_time_to_wait < self.max_time: + time.sleep(min_time_to_wait) + self.check_all_tokens() + else: + raise GitHubAPILimitException(f"Rate limits for all tokens ({count_attr}) were reached") + else: + self.update_token() + return False diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index c3d9c1c98188f9..46226f1d020efb 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -2,4 +2,28 @@ import os +import pytest +import responses + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" + + +@pytest.fixture(name="rate_limit_mock_response") +def rate_limit_mock_response(): + rate_limit_response = { + "resources": { + "core": { + "limit": 5000, + "used": 0, + "remaining": 5000, + "reset": 4070908800 + }, + "graphql": { + "limit": 5000, + "used": 0, + "remaining": 5000, + "reset": 4070908800 + } + } + } + responses.add(responses.GET, "https://api.github.com/rate_limit", json=rate_limit_response) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py new file mode 100644 index 00000000000000..e8bb59250b5115 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from unittest.mock import patch + +import pendulum +import responses +from freezegun import freeze_time +from source_github import SourceGithub +from source_github.streams import Organizations +from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter, read_full_refresh + + +@responses.activate +def test_multiple_tokens(rate_limit_mock_response): + authenticator = SourceGithub()._get_authenticator({"access_token": "token_1, token_2, token_3"}) + assert isinstance(authenticator, MultipleTokenAuthenticatorWithRateLimiter) + assert ["token_1", "token_2", "token_3"] == list(authenticator._tokens) + + +@responses.activate +def test_authenticator_counter(rate_limit_mock_response): + """ + This test ensures that the rate limiter: + 1. correctly handles the available limits from GitHub API and saves it. + 2. correctly counts the number of requests made. + """ + authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"]) + + assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(5000, 5000), (5000, 5000), (5000, 5000)] + organization_args = {"organizations": ["org1", "org2"], "authenticator": authenticator} + stream = Organizations(**organization_args) + responses.add("GET", "https://api.github.com/orgs/org1", json={"id": 1}) + responses.add("GET", "https://api.github.com/orgs/org2", json={"id": 2}) + list(read_full_refresh(stream)) + assert authenticator._tokens["token1"].count_rest == 4998 + + +@responses.activate +def test_multiple_token_authenticator_with_rate_limiter(caplog): + """ + This test ensures that: + 1. The rate limiter iterates over all tokens one-by-one after the previous is fully drained. + 2. Counter is set to zero after 1500 requests were made. (500 available requests per key were set as default) + 3. Exception is handled and log warning message could be found in output. Connector does not raise AirbyteTracedException because there might be GraphQL streams with remaining request we still can read. + """ + + counter_rate_limits = 0 + counter_orgs = 0 + + def request_callback_rate_limits(request): + nonlocal counter_rate_limits + while counter_rate_limits < 3: + counter_rate_limits += 1 + resp_body = { + "resources": { + "core": { + "limit": 500, + "used": 0, + "remaining": 500, + "reset": 4070908800 + }, + "graphql": { + "limit": 500, + "used": 0, + "remaining": 500, + "reset": 4070908800 + } + } + } + return (200, {}, json.dumps(resp_body)) + + responses.add_callback(responses.GET, "https://api.github.com/rate_limit", callback=request_callback_rate_limits) + authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"]) + organization_args = {"organizations": ["org1"], "authenticator": authenticator} + stream = Organizations(**organization_args) + + def request_callback_orgs(request): + nonlocal counter_orgs + while counter_orgs < 1_501: + counter_orgs += 1 + resp_body = {"id": 1} + headers = {"Link": '; rel="next"'} + return (200, headers, json.dumps(resp_body)) + + responses.add_callback( + responses.GET, + "https://api.github.com/orgs/org1", + callback=request_callback_orgs, + content_type="application/json", + ) + + list(read_full_refresh(stream)) + assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(0, 500), (0, 500), (0, 500)] + assert "Limits for all provided tokens are reached, please try again later" in caplog.messages + + +@freeze_time("2021-01-01 12:00:00") +@responses.activate +@patch("time.sleep") +def test_multiple_token_authenticator_with_rate_limiter_and_sleep(sleep_mock, caplog): + """ + This test ensures that: + 1. The rate limiter will only wait (sleep) for token availability if the nearest available token appears within 600 seconds (see max_time). + 2. Token Counter is reset to new values after 1500 requests were made and last token is still in use. + """ + + counter_rate_limits = 0 + counter_orgs = 0 + ACCEPTED_WAITING_TIME_IN_SECONDS = 595 + reset_time = (pendulum.now() + pendulum.duration(seconds=ACCEPTED_WAITING_TIME_IN_SECONDS)).int_timestamp + + def request_callback_rate_limits(request): + nonlocal counter_rate_limits + while counter_rate_limits < 6: + counter_rate_limits += 1 + resp_body = { + "resources": { + "core": { + "limit": 500, + "used": 0, + "remaining": 500, + "reset": reset_time + }, + "graphql": { + "limit": 500, + "used": 0, + "remaining": 500, + "reset": reset_time + } + } + } + return (200, {}, json.dumps(resp_body)) + + responses.add_callback(responses.GET, "https://api.github.com/rate_limit", callback=request_callback_rate_limits) + authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"]) + organization_args = {"organizations": ["org1"], "authenticator": authenticator} + stream = Organizations(**organization_args) + + def request_callback_orgs(request): + nonlocal counter_orgs + while counter_orgs < 1_501: + counter_orgs += 1 + resp_body = {"id": 1} + headers = {"Link": '; rel="next"'} + return (200, headers, json.dumps(resp_body)) + return (200, {}, json.dumps({"id": 2})) + + responses.add_callback( + responses.GET, + "https://api.github.com/orgs/org1", + callback=request_callback_orgs, + content_type="application/json", + ) + + list(read_full_refresh(stream)) + sleep_mock.assert_called_once_with(ACCEPTED_WAITING_TIME_IN_SECONDS) + assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(500, 500), (500, 500), (498, 500)] diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py index 8ec9d79c574d0f..71810c347480e3 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py @@ -2,20 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime import logging import os -import time from unittest.mock import MagicMock import pytest import responses from airbyte_cdk.models import AirbyteConnectionStatus, Status from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from freezegun import freeze_time from source_github import constants from source_github.source import SourceGithub -from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter from .utils import command_check @@ -42,7 +38,7 @@ def check_source(repo_line: str) -> AirbyteConnectionStatus: ({"access_token": "test_token", "repository": "airbyte/test"}, True), ), ) -def test_check_start_date(config, expected): +def test_check_start_date(config, expected, rate_limit_mock_response): responses.add(responses.GET, "https://api.github.com/repos/airbyte/test?per_page=100", json={"full_name": "test_full_name"}) source = SourceGithub() status, _ = source.check_connection(logger=logging.getLogger("airbyte"), config=config) @@ -73,18 +69,18 @@ def test_connection_fail_due_to_config_error(api_url, deployment_env, expected_m @responses.activate -def test_check_connection_repos_only(): +def test_check_connection_repos_only(rate_limit_mock_response): responses.add("GET", "https://api.github.com/repos/airbytehq/airbyte", json={"full_name": "airbytehq/airbyte"}) status = check_source("airbytehq/airbyte airbytehq/airbyte airbytehq/airbyte") assert not status.message assert status.status == Status.SUCCEEDED # Only one request since 3 repos have same name - assert len(responses.calls) == 1 + assert len(responses.calls) == 2 @responses.activate -def test_check_connection_repos_and_org_repos(): +def test_check_connection_repos_and_org_repos(rate_limit_mock_response): repos = [{"name": f"name {i}", "full_name": f"full name {i}", "updated_at": "2020-01-01T00:00:00Z"} for i in range(1000)] responses.add( "GET", "https://api.github.com/repos/airbyte/test", json={"full_name": "airbyte/test", "organization": {"login": "airbyte"}} @@ -99,11 +95,11 @@ def test_check_connection_repos_and_org_repos(): assert not status.message assert status.status == Status.SUCCEEDED # Two requests for repos and two for organization - assert len(responses.calls) == 4 + assert len(responses.calls) == 5 @responses.activate -def test_check_connection_org_only(): +def test_check_connection_org_only(rate_limit_mock_response): repos = [{"name": f"name {i}", "full_name": f"full name {i}", "updated_at": "2020-01-01T00:00:00Z"} for i in range(1000)] responses.add("GET", "https://api.github.com/orgs/airbytehq/repos", json=repos) @@ -111,7 +107,7 @@ def test_check_connection_org_only(): assert not status.message assert status.status == Status.SUCCEEDED # One request to check organization - assert len(responses.calls) == 1 + assert len(responses.calls) == 2 @responses.activate @@ -183,7 +179,8 @@ def test_get_org_repositories(): assert set(organisations) == {"airbytehq", "docker"} -def test_organization_or_repo_available(monkeypatch): +@responses.activate +def test_organization_or_repo_available(monkeypatch, rate_limit_mock_response): monkeypatch.setattr(SourceGithub, "_get_org_repositories", MagicMock(return_value=(False, False))) source = SourceGithub() with pytest.raises(Exception) as exc_info: @@ -238,55 +235,16 @@ def test_check_config_repository(): assert command_check(source, config) -def test_streams_no_streams_available_error(monkeypatch): +@responses.activate +def test_streams_no_streams_available_error(monkeypatch, rate_limit_mock_response): monkeypatch.setattr(SourceGithub, "_get_org_repositories", MagicMock(return_value=(False, False))) with pytest.raises(AirbyteTracedException) as e: SourceGithub().streams(config={"access_token": "test_token", "repository": "airbytehq/airbyte-test"}) assert str(e.value) == "No streams available. Please check permissions" -def test_multiple_token_authenticator_with_rate_limiter(monkeypatch): - - called_args = [] - - def sleep_mock(seconds): - frozen_time.tick(delta=datetime.timedelta(seconds=seconds)) - called_args.append(seconds) - - monkeypatch.setattr(time, "sleep", sleep_mock) - - with freeze_time("2021-01-01 12:00:00") as frozen_time: - - authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2"], requests_per_hour=4) - authenticator._tokens["token1"].count = 2 - - assert authenticator.token == "Bearer token1" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token1" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - - # token1 is fully exhausted, token2 is still used - assert authenticator._tokens["token1"].count == 0 - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert called_args == [] - - # now we have to sleep because all tokens are exhausted - assert authenticator.token == "Bearer token1" - assert called_args == [3594.0] - - assert authenticator._tokens["token1"].count == 3 - assert authenticator._tokens["token2"].count == 4 - - @responses.activate -def test_streams_page_size(): +def test_streams_page_size(rate_limit_mock_response): responses.get("https://api.github.com/repos/airbytehq/airbyte", json={"full_name": "airbytehq/airbyte", "default_branch": "master"}) responses.get("https://api.github.com/repos/airbytehq/airbyte/branches", json=[{"repository": "airbytehq/airbyte", "name": "master"}]) @@ -322,7 +280,7 @@ def test_streams_page_size(): ({"access_token": "test_token", "repository": "airbyte/test"}, 39), ), ) -def test_streams_config_start_date(config, expected): +def test_streams_config_start_date(config, expected, rate_limit_mock_response): responses.add(responses.GET, "https://api.github.com/repos/airbyte/test?per_page=100", json={"full_name": "airbyte/test"}) responses.add( responses.GET, diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index 87d9c3478cd33d..3a8bc857f032d3 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -234,6 +234,7 @@ def test_stream_organizations_read(): def test_stream_teams_read(): organization_args = {"organizations": ["org1", "org2"]} stream = Teams(**organization_args) + stream._session.cache.clear() responses.add("GET", "https://api.github.com/orgs/org1/teams", json=[{"id": 1}, {"id": 2}]) responses.add("GET", "https://api.github.com/orgs/org2/teams", json=[{"id": 3}]) records = list(read_full_refresh(stream)) @@ -533,7 +534,8 @@ def test_stream_project_columns(): projects_stream = Projects(**repository_args_with_start_date) stream = ProjectColumns(projects_stream, **repository_args_with_start_date) - + projects_stream._session.cache.clear() + stream._session.cache.clear() stream_state = {} records = read_incremental(stream, stream_state=stream_state) @@ -918,7 +920,7 @@ def request_callback(request): @responses.activate -def test_stream_team_members_full_refresh(caplog): +def test_stream_team_members_full_refresh(caplog, rate_limit_mock_response): organization_args = {"organizations": ["org1"]} repository_args = {"repositories": [], "page_size_for_large_streams": 100} @@ -959,6 +961,7 @@ def test_stream_commit_comment_reactions_incremental_read(): repository_args = {"repositories": ["airbytehq/integration-test"], "page_size_for_large_streams": 100} stream = CommitCommentReactions(**repository_args) + stream._parent_stream._session.cache.clear() responses.add( "GET", @@ -1305,7 +1308,7 @@ def request_callback(request): @responses.activate -def test_stream_projects_v2_graphql_retry(): +def test_stream_projects_v2_graphql_retry(rate_limit_mock_response): repository_args_with_start_date = { "start_date": "2022-01-01T00:00:00Z", "page_size_for_large_streams": 20, @@ -1368,7 +1371,7 @@ def test_stream_contributor_activity_parse_empty_response(caplog): @responses.activate -def test_stream_contributor_activity_accepted_response(caplog): +def test_stream_contributor_activity_accepted_response(caplog, rate_limit_mock_response): responses.add( responses.GET, "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100", diff --git a/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py deleted file mode 100644 index e7e3adf6bf81d5..00000000000000 --- a/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.sources.streams.http.auth import MultipleTokenAuthenticator -from source_github import SourceGithub - - -def test_single_token(): - authenticator = SourceGithub()._get_authenticator({"access_token": "123"}) - assert isinstance(authenticator, MultipleTokenAuthenticator) - assert ["123"] == authenticator._tokens - authenticator = SourceGithub()._get_authenticator({"credentials": {"access_token": "123"}}) - assert ["123"] == authenticator._tokens - authenticator = SourceGithub()._get_authenticator({"credentials": {"personal_access_token": "123"}}) - assert ["123"] == authenticator._tokens - - -def test_multiple_tokens(): - authenticator = SourceGithub()._get_authenticator({"access_token": "123, 456"}) - assert isinstance(authenticator, MultipleTokenAuthenticator) - assert ["123", "456"] == authenticator._tokens diff --git a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml index 63c87ce3137f5e..d0e00aadd2800e 100644 --- a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml @@ -6,6 +6,9 @@ acceptance_tests: - config_path: secrets/service_config.json expect_records: path: integration_tests/expected_records.txt + file_types: + skip_test: true + bypass_reason: "The source only supports Google Sheets" connection: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl index d5338e2d7ffd47..62e84cc8343352 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl @@ -21,9 +21,8 @@ {"stream": "groups", "data": {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87"}, "emitted_at": 1697453247031} {"stream": "groups", "data": {"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1"}, "emitted_at": 1697453247032} {"stream": "groups", "data": {"name": "jira-servicemanagement-customers-airbyteio", "groupId": "125680d3-7e85-41ad-a662-892b6590272e"}, "emitted_at": 1697453247033} -{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10625", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "key": "IT-25", "renderedFields": {"statuscategorychangedate": "17/May/22 4:06 AM", "created": "17/May/22 4:06 AM", "customfield_10017": "dark_yellow", "updated": "17/May/22 4:28 AM", "description": "

Implement OAUth

", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {}, "attachment": [], "environment": "", "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "17/May/22 4:06 AM", "updated": "17/May/22 4:06 AM", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 1, "total": 1, "histories": [{"id": "15129", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-26", "toString": "This issue is cloned by IT-26"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:06:24.675-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "workratio": -1, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "created": "2022-05-17T04:06:24.048000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10019": "0|i0076v:", "customfield_10217": [], "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "inwardIssue": {"id": "10626", "key": "IT-26", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "fields": {"summary": "CLONE - Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2022-05-17T04:28:19.876000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {}, "attachment": [], "summary": "Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "progress": {"progress": 0, "total": 0}, "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076-0700", "updated": "2022-05-17T04:06:55.076-0700", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:06:24.048000-07:00", "updated": "2022-05-17T04:28:19.876000-07:00"}, "emitted_at": 1701283916831} -{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10080", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080", "key": "IT-24", "renderedFields": {"statuscategorychangedate": "11/Mar/21 6:17 AM", "timespent": "5 hours, 48 minutes", "aggregatetimespent": "5 hours, 48 minutes", "created": "11/Mar/21 6:17 AM", "customfield_10017": "", "timeestimate": "0 minutes", "updated": "05/Apr/23 4:58 AM", "description": "

Test description 74

", "timetracking": {"remainingEstimate": "0 minutes", "timeSpent": "5 hours, 48 minutes", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "14/Apr/21 2:11 PM", "size": "7 kB", "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": "0 minutes", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 21 minutes", "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "37 minutes", "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 50 minutes", "id": "11710", "issueId": "10080"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 8, "total": 8, "histories": [{"id": "15179", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-04-05T04:58:35.333-0700", "items": [{"field": "Sprint", "fieldtype": "custom", "fieldId": "customfield_10020", "from": "", "fromString": "", "to": "10", "toString": "IT Sprint 9"}]}, {"id": "14989", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.917-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "10680", "fromString": "10680", "to": "20880", "toString": "20880"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11710", "toString": "11710"}]}, {"id": "14988", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.314-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "8460", "fromString": "8460", "to": "10680", "toString": "10680"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11709", "toString": "11709"}]}, {"id": "14987", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:46.691-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "8460", "toString": "8460"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11708", "toString": "11708"}]}, {"id": "14800", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T07:18:07.884-0700", "items": [{"field": "RemoteIssueLink", "fieldtype": "jira", "from": null, "fromString": null, "to": "10046", "toString": "This issue links to \"TSTSUP-111 (My Acme Tracker)\""}]}, {"id": "14718", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:54.455-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-22", "toString": "This issue is duplicated by IT-22"}]}, {"id": "14716", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-23", "toString": "This issue is duplicated by IT-23"}]}, {"id": "14596", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.899-0700", "items": [{"field": "Attachment", "fieldtype": "jira", "fieldId": "attachment", "from": null, "fromString": null, "to": "10123", "toString": "demo.xlsx"}]}]}, "fields": {"statuscategorychangedate": "2021-03-11T06:17:33.483-0800", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}, "timespent": 20880, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 20880, "workratio": -1, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/watchers", "watchCount": 1, "isWatching": true}, "created": "2021-03-11T06:17:33.169000-08:00", "customfield_10020": [{"id": 10, "name": "IT Sprint 9", "state": "future", "boardId": 1, "startDate": "2022-09-06T11:25:59.072Z", "endDate": "2022-09-20T11:25:00.000Z"}], "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "labels": [], "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i000hr:", "timeestimate": 0, "versions": [], "issuelinks": [{"id": "10244", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10244", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10069", "key": "IT-22", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10069", "fields": {"summary": "Test 63", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}, {"id": "10243", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10243", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10075", "key": "IT-23", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075", "fields": {"summary": "Test 69", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}}}}], "updated": "2023-04-05T04:58:35.329000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [], "description": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test description 74"}]}]}, "timetracking": {"remainingEstimate": "0m", "timeSpent": "5h 48m", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.652-0700", "size": 7360, "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": 0, "summary": "Test 74", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 20880, "total": 20880, "percent": 100}, "progress": {"progress": 20880, "total": 20880, "percent": 100}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/votes", "votes": 1, "hasVoted": true}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:46.574-0700", "updated": "2021-04-15T11:39:46.574-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 21m", "timeSpentSeconds": 8460, "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215-0700", "updated": "2021-04-15T11:39:47.215-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 2", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.834-0700", "updated": "2021-04-15T11:39:47.834-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 50m", "timeSpentSeconds": 10200, "id": "11710", "issueId": "10080"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2021-03-11T06:17:33.169000-08:00", "updated": "2023-04-05T04:58:35.329000-07:00"}, "emitted_at": 1701283916937} -{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10626", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "key": "IT-26", "renderedFields": {"statuscategorychangedate": "17/May/22 4:28 AM", "timespent": "1 day", "aggregatetimespent": "1 day", "lastViewed": "12/Oct/23 1:43 PM", "created": "17/May/22 4:28 AM", "customfield_10017": "dark_yellow", "timeestimate": "1 week, 1 day", "aggregatetimeoriginalestimate": "2 weeks, 4 days, 5 hours", "updated": "12/Oct/23 1:43 PM", "timeoriginalestimate": "2 weeks, 4 days, 5 hours", "description": "

Implement OAUth

", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2 weeks, 4 days, 5 hours", "remainingEstimate": "1 week, 1 day", "timeSpent": "1 day", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": "1 week, 1 day", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "05/Apr/23 5:08 AM", "updated": "05/Apr/23 5:08 AM", "started": "05/Apr/23 1:00 AM", "timeSpent": "1 day", "id": "11820", "issueId": "10626"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 4, "total": 4, "histories": [{"id": "15198", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:15.036-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "172800", "toString": "172800"}]}, {"id": "15197", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:05.182-0700", "items": [{"field": "timeoriginalestimate", "fieldtype": "jira", "fieldId": "timeoriginalestimate", "from": null, "fromString": null, "to": "421200", "toString": "421200"}]}, {"id": "15186", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "2023-04-05T05:08:50.115-0700", "items": [{"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "28800", "toString": "28800"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11820", "toString": "11820"}]}, {"id": "15128", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.837-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-25", "toString": "This issue clones IT-25"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:28:19.775-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": 28800, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 28800, "workratio": 6, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "lastViewed": "2023-10-12T13:43:22.992-0700", "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "watchCount": 1, "isWatching": true}, "created": "2022-05-17T04:28:19.523000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i00773:", "timeestimate": 172800, "aggregatetimeoriginalestimate": 421200, "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "outwardIssue": {"id": "10625", "key": "IT-25", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "fields": {"summary": "Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-10-12T13:43:15.025000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": 421200, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2w 4d 5h", "remainingEstimate": "1w 1d", "timeSpent": "1d", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": 172800, "summary": "CLONE - Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 28800, "total": 201600, "percent": 14}, "progress": {"progress": 28800, "total": 201600, "percent": 14}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:28:19.523000-07:00", "updated": "2023-10-12T13:43:15.025000-07:00"}, "emitted_at": 1701283916963} +{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10080", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080", "key": "IT-24", "renderedFields": {"statuscategorychangedate": "11/Mar/21 6:17 AM", "timespent": "5 hours, 48 minutes", "aggregatetimespent": "5 hours, 48 minutes", "created": "11/Mar/21 6:17 AM", "customfield_10017": "", "timeestimate": "0 minutes", "updated": "05/Apr/23 4:58 AM", "description": "

Test description 74

", "timetracking": {"remainingEstimate": "0 minutes", "timeSpent": "5 hours, 48 minutes", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "14/Apr/21 2:11 PM", "size": "7 kB", "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": "0 minutes", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 21 minutes", "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "37 minutes", "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 50 minutes", "id": "11710", "issueId": "10080"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 8, "total": 8, "histories": [{"id": "15179", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-04-05T04:58:35.333-0700", "items": [{"field": "Sprint", "fieldtype": "custom", "fieldId": "customfield_10020", "from": "", "fromString": "", "to": "10", "toString": "IT Sprint 9"}]}, {"id": "14989", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.917-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "10680", "fromString": "10680", "to": "20880", "toString": "20880"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11710", "toString": "11710"}]}, {"id": "14988", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.314-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "8460", "fromString": "8460", "to": "10680", "toString": "10680"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11709", "toString": "11709"}]}, {"id": "14987", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:46.691-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "8460", "toString": "8460"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11708", "toString": "11708"}]}, {"id": "14800", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T07:18:07.884-0700", "items": [{"field": "RemoteIssueLink", "fieldtype": "jira", "from": null, "fromString": null, "to": "10046", "toString": "This issue links to \"TSTSUP-111 (My Acme Tracker)\""}]}, {"id": "14718", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:54.455-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-22", "toString": "This issue is duplicated by IT-22"}]}, {"id": "14716", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-23", "toString": "This issue is duplicated by IT-23"}]}, {"id": "14596", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.899-0700", "items": [{"field": "Attachment", "fieldtype": "jira", "fieldId": "attachment", "from": null, "fromString": null, "to": "10123", "toString": "demo.xlsx"}]}]}, "fields": {"statuscategorychangedate": "2021-03-11T06:17:33.483-0800", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}, "timespent": 20880, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 20880, "workratio": -1, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "created": "2021-03-11T06:17:33.169000-08:00", "customfield_10020": [{"id": 10, "name": "IT Sprint 9", "state": "future", "boardId": 1, "startDate": "2022-09-06T11:25:59.072Z", "endDate": "2022-09-20T11:25:00.000Z"}], "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "labels": [], "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10019": "0|i000hr:", "customfield_10217": [], "timeestimate": 0, "versions": [], "issuelinks": [{"id": "10244", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10244", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10069", "key": "IT-22", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10069", "fields": {"summary": "Test 63", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}, {"id": "10243", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10243", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10075", "key": "IT-23", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075", "fields": {"summary": "Test 69", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}}}}], "updated": "2023-04-05T04:58:35.329000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [], "description": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test description 74"}]}]}, "timetracking": {"remainingEstimate": "0m", "timeSpent": "5h 48m", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.652-0700", "size": 7360, "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": 0, "summary": "Test 74", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 20880, "total": 20880, "percent": 100}, "progress": {"progress": 20880, "total": 20880, "percent": 100}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/votes", "votes": 1, "hasVoted": true}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:46.574-0700", "updated": "2021-04-15T11:39:46.574-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 21m", "timeSpentSeconds": 8460, "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215-0700", "updated": "2021-04-15T11:39:47.215-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 2", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.834-0700", "updated": "2021-04-15T11:39:47.834-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 50m", "timeSpentSeconds": 10200, "id": "11710", "issueId": "10080"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2021-03-11T06:17:33.169000-08:00", "updated": "2023-04-05T04:58:35.329000-07:00"}, "emitted_at": 1706087956389} +{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10626", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "key": "IT-26", "renderedFields": {"statuscategorychangedate": "17/May/22 4:28 AM", "timespent": "1 day", "aggregatetimespent": "1 day", "created": "17/May/22 4:28 AM", "customfield_10017": "dark_yellow", "timeestimate": "1 week, 1 day", "aggregatetimeoriginalestimate": "2 weeks, 4 days, 5 hours", "updated": "12/Oct/23 1:43 PM", "timeoriginalestimate": "2 weeks, 4 days, 5 hours", "description": "

Implement OAUth

", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2 weeks, 4 days, 5 hours", "remainingEstimate": "1 week, 1 day", "timeSpent": "1 day", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": "1 week, 1 day", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "05/Apr/23 5:08 AM", "updated": "05/Apr/23 5:08 AM", "started": "05/Apr/23 1:00 AM", "timeSpent": "1 day", "id": "11820", "issueId": "10626"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 4, "total": 4, "histories": [{"id": "15198", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:15.036-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "172800", "toString": "172800"}]}, {"id": "15197", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:05.182-0700", "items": [{"field": "timeoriginalestimate", "fieldtype": "jira", "fieldId": "timeoriginalestimate", "from": null, "fromString": null, "to": "421200", "toString": "421200"}]}, {"id": "15186", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "2023-04-05T05:08:50.115-0700", "items": [{"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "28800", "toString": "28800"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11820", "toString": "11820"}]}, {"id": "15128", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.837-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-25", "toString": "This issue clones IT-25"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:28:19.775-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": 28800, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 28800, "workratio": 6, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "watchCount": 1, "isWatching": true}, "created": "2022-05-17T04:28:19.523000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i00773:", "timeestimate": 172800, "aggregatetimeoriginalestimate": 421200, "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "outwardIssue": {"id": "10625", "key": "IT-25", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "fields": {"summary": "Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-10-12T13:43:15.025000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": 421200, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2w 4d 5h", "remainingEstimate": "1w 1d", "timeSpent": "1d", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": 172800, "summary": "CLONE - Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 28800, "total": 201600, "percent": 14}, "progress": {"progress": 28800, "total": 201600, "percent": 14}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:28:19.523000-07:00", "updated": "2023-10-12T13:43:15.025000-07:00"}, "emitted_at": 1706087956632} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076000-07:00", "updated": "2022-05-17T04:06:55.076000-07:00", "jsdPublic": true, "issueId": "IT-25"}, "emitted_at": 1697453253441} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10521", "id": "10521", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque eget venenatis elit. Duis eu justo eget augue iaculis fermentum. Sed semper quam laoreet nisi egestas at posuere augue semper.", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:32:43.099000-07:00", "updated": "2021-04-14T14:32:43.099000-07:00", "jsdPublic": true, "issueId": "IT-23"}, "emitted_at": 1697453254086} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10639", "id": "10639", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Linked related issue!", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.998000-07:00", "updated": "2021-04-15T00:08:48.998000-07:00", "jsdPublic": true, "issueId": "IT-23"}, "emitted_at": 1697453254087} diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml index a46d1dad743d6f..f10331aafc5ddd 100644 --- a/airbyte-integrations/connectors/source-jira/metadata.yaml +++ b/airbyte-integrations/connectors/source-jira/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 - dockerImageTag: 1.0.0 + dockerImageTag: 1.0.1 dockerRepository: airbyte/source-jira documentationUrl: https://docs.airbyte.com/integrations/sources/jira githubIssueLabel: source-jira diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 542ea67ea59d87..05c0d838114996 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -48,12 +48,14 @@ class JiraStream(HttpStream, ABC): extract_field: Optional[str] = None api_v1 = False # Defines the HTTP status codes for which the slice should be skipped. - # Refernce issue: https://github.com/airbytehq/oncall/issues/2133 + # Reference issue: https://github.com/airbytehq/oncall/issues/2133 # we should skip the slice with `board id` which doesn't support `sprints` # it's generally applied to all streams that might have the same error hit in the future. skip_http_status_codes = [requests.codes.BAD_REQUEST] raise_on_http_errors = True transformer: TypeTransformer = DateTimeTransformer(TransformConfig.DefaultSchemaNormalization) + # emitting state message after every page read + state_checkpoint_interval = page_size def __init__(self, domain: str, projects: List[str], **kwargs): super().__init__(**kwargs) @@ -254,7 +256,6 @@ class BoardIssues(StartDateJiraStream): cursor_field = "updated" extract_field = "issues" api_v1 = True - state_checkpoint_interval = 50 # default page size is 50 def __init__(self, **kwargs): super().__init__(**kwargs) @@ -416,7 +417,6 @@ class Issues(IncrementalJiraStream): # Issue: https://github.com/airbytehq/airbyte/issues/26712 # we should skip the slice with wrong permissions on project level skip_http_status_codes = [requests.codes.FORBIDDEN, requests.codes.BAD_REQUEST] - state_checkpoint_interval = 50 # default page size is 50 def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index ed08fabcc6837f..f10438c5208d6b 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -43,7 +43,8 @@ dependencies { integrationTestJavaImplementation libs.apache.commons.lang integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mongodb-v2') - dataGeneratorImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + dataGeneratorImplementation platform(libs.fasterxml) + dataGeneratorImplementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310' dataGeneratorImplementation project(':airbyte-integrations:connectors:source-mongodb-v2') dataGeneratorImplementation libs.mongo.driver.sync diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt b/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt index d80a179a5947c4..fd2b7f612930d0 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt @@ -1,7 +1,10 @@ package io.airbyte.integrations.source.mongodb +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.github.javafaker.Faker -import io.airbyte.commons.json.Jsons import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.cli.ArgParser import kotlinx.cli.ArgType @@ -31,7 +34,7 @@ object MongoDbInsertClient { println("Enter password: ") val password = readln() - var config = mapOf(MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY to + val config = mapOf(MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY to mapOf( MongoConstants.DATABASE_CONFIGURATION_KEY to databaseName, MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY to connectionString, @@ -42,7 +45,12 @@ object MongoDbInsertClient { val faker = Faker(); - MongoConnectionUtils.createMongoClient(MongoDbSourceConfig(Jsons.deserialize(Jsons.serialize(config)))).use { mongoClient -> + val objectMapper = ObjectMapper().registerModule(JavaTimeModule()) + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + objectMapper.configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true) + val roundTrippedConfig = objectMapper.readTree(objectMapper.writeValueAsBytes(config)) + + MongoConnectionUtils.createMongoClient(MongoDbSourceConfig(roundTrippedConfig)).use { mongoClient -> val documents = mutableListOf() val batches = if (numberOfDocuments > BATCH_SIZE) numberOfDocuments / BATCH_SIZE else 1; val batchSize = if (numberOfDocuments > BATCH_SIZE) BATCH_SIZE else numberOfDocuments; diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 6fadebd3ed3727..14c4cd1da3b4ec 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -13,7 +13,7 @@ java { } airbyteJavaConnector { - cdkVersionRequired = '0.11.5' + cdkVersionRequired = '0.14.0' features = ['db-sources'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index 23d5517fe512a7..482126b1055e0c 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.3.1 + dockerImageTag: 3.3.2 dockerRepository: airbyte/source-postgres documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index fd4573d7424b38..ded9f8c130cee6 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -145,7 +145,6 @@ public class PostgresSource extends AbstractJdbcSource implements private Set publicizedTablesInCdc; private static final Set INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer"); private int stateEmissionFrequency; - private XminStatus xminStatus; public static Source sshWrappedSource(PostgresSource source) { return new SshWrappedSource(source, JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, "security"); @@ -273,11 +272,6 @@ protected void logPreSyncDebugData(final JdbcDatabase database, final Configured indexInfo.close(); } - // Log and save the xmin status - this.xminStatus = PostgresQueryUtils.getXminStatus(database); - LOGGER.info(String.format("Xmin Status : {Number of wraparounds: %s, Xmin Transaction Value: %s, Xmin Raw Value: %s", - xminStatus.getNumWraparound(), xminStatus.getXminXidValue(), xminStatus.getXminRawValue())); - } @Override @@ -483,6 +477,15 @@ public List> getIncrementalIterators(final } if (isAnyStreamIncrementalSyncMode(catalog) && PostgresUtils.isXmin(sourceConfig)) { + // Log and save the xmin status + final XminStatus xminStatus; + try { + xminStatus = PostgresQueryUtils.getXminStatus(database); + } catch (SQLException e) { + throw new RuntimeException(e); + } + LOGGER.info(String.format("Xmin Status : {Number of wraparounds: %s, Xmin Transaction Value: %s, Xmin Raw Value: %s", + xminStatus.getNumWraparound(), xminStatus.getXminXidValue(), xminStatus.getXminRawValue())); final StreamsCategorised streamsCategorised = categoriseStreams(stateManager, catalog, xminStatus); final ResultWithFailed> streamsUnderVacuum = streamsUnderVacuum(database, streamsCategorised.ctidStreams().streamsForCtidSync(), diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml index 4d8db46aba243c..7b253a10ef6223 100644 --- a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml @@ -6,105 +6,168 @@ acceptance_tests: path: integration_tests/expected_records/csv.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/config_iam_role.json expect_records: path: integration_tests/expected_records/csv.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_custom_encoding_config.json expect_records: path: integration_tests/expected_records/legacy_csv_custom_encoding.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_custom_format_config.json expect_records: path: integration_tests/expected_records/legacy_csv_custom_format.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_user_schema_config.json expect_records: path: integration_tests/expected_records/legacy_csv_user_schema.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_no_header_config.json expect_records: path: integration_tests/expected_records/legacy_csv_no_header.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_skip_rows_config.json expect_records: path: integration_tests/expected_records/legacy_csv_skip_rows.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_skip_rows_no_header_config.json expect_records: path: integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_with_nulls_config.json expect_records: path: integration_tests/expected_records/legacy_csv_with_nulls.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_with_null_bools_config.json expect_records: path: integration_tests/expected_records/legacy_csv_with_null_bools.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_parquet_config.json expect_records: path: integration_tests/expected_records/parquet.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/parquet_dataset_config.json expect_records: path: integration_tests/expected_records/parquet_dataset.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_parquet_decimal_config.json expect_records: path: integration_tests/expected_records/legacy_parquet_decimal.jsonl timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_avro_config.json expect_records: path: integration_tests/expected_records/avro.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_jsonl_config.json expect_records: path: integration_tests/expected_records/jsonl.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_jsonl_newlines_config.json expect_records: path: integration_tests/expected_records/jsonl_newlines.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_csv.json expect_records: path: integration_tests/expected_records/zip_csv.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_csv_custom_encoding.json expect_records: path: integration_tests/expected_records/zip_csv_custom_encoding.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_jsonl.json expect_records: path: integration_tests/expected_records/zip_jsonl.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_avro.json expect_records: path: integration_tests/expected_records/zip_avro.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_parquet.json expect_records: path: integration_tests/expected_records/zip_parquet.jsonl exact_order: true timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/unstructured_config.json expect_records: path: integration_tests/expected_records/unstructured.jsonl diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl index d3b48ade375a5e..25517e84945e2f 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl @@ -5,8 +5,8 @@ {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "object": "setup_attempt", "application": null, "created": 1649752931, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627242509} {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "object": "setup_attempt", "application": null, "created": 1649752923, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627243547} {"stream": "accounts", "data": {"id": "acct_1NGp6SD04fX0Aizk", "object": "account", "capabilities": {"acss_debit_payments": "active", "affirm_payments": "active", "afterpay_clearpay_payments": "active", "bancontact_payments": "active", "card_payments": "active", "cartes_bancaires_payments": "pending", "cashapp_payments": "active", "eps_payments": "active", "giropay_payments": "active", "ideal_payments": "active", "klarna_payments": "active", "link_payments": "active", "p24_payments": "active", "sepa_debit_payments": "active", "sofort_payments": "active", "transfers": "active", "us_bank_account_ach_payments": "active"}, "charges_enabled": true, "country": "US", "default_currency": "usd", "details_submitted": true, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": true, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": "AIRBYTE", "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": "Airbyte", "timezone": "Asia/Tbilisi"}, "payments": {"statement_descriptor": "WWW.AIRBYTE.COM", "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267880} -{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1697627267882} -{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267884} +{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"annual_revenue": null,"estimated_worker_count": null,"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1697627267882} +{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"annual_revenue": null,"estimated_worker_count": null,"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267884} {"stream": "shipping_rates", "data": {"id": "shr_1NXgplEcXtiJtvvhA1ntV782", "object": "shipping_rate", "active": true, "created": 1690274589, "delivery_estimate": "{'maximum': {'unit': 'business_day', 'value': 14}, 'minimum': {'unit': 'business_day', 'value': 10}}", "display_name": "Test Ground Shipping", "fixed_amount": {"amount": 999, "currency": "usd"}, "livemode": false, "metadata": {}, "tax_behavior": "inclusive", "tax_code": "txcd_92010001", "type": "fixed_amount"}, "emitted_at": 1697627269309} {"stream": "balance_transactions", "data": {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": -9164, "available_on": 1645488000, "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "exchange_rate": null, "fee": 0, "fee_details": [], "net": -9164, "reporting_category": "payout", "source": "po_1KVQhfEcXtiJtvvhZlUkl08U", "status": "available", "type": "payout"}, "emitted_at": 1697627270253} {"stream": "balance_transactions", "data": {"id": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "object": "balance_transaction", "amount": 5300, "available_on": 1640649600, "created": 1640120473, "currency": "usd", "description": null, "exchange_rate": null, "fee": 184, "fee_details": [{"amount": 184, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 5116, "reporting_category": "charge", "source": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "status": "available", "type": "charge"}, "emitted_at": 1697627270254} @@ -17,7 +17,7 @@ {"stream": "file_links", "data": {"id": "link_1KnfIiEcXtiJtvvhCNceSyei", "object": "file_link", "created": 1649752936, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfY1FvanBFTmt0dUdrRWJXTHBpUlVYVUtu007305bsv3"}, "emitted_at": 1697627273833} {"stream": "file_links", "data": {"id": "link_1KnfIbEcXtiJtvvhyBLUqkSt", "object": "file_link", "created": 1649752929, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfaXh1blBqMmY0MzI3SHZWbUZIeFVGU3Nl0022JjupYq"}, "emitted_at": 1697627273834} {"stream": "file_links", "data": {"id": "link_1KnfIUEcXtiJtvvh0ktKHfWz", "object": "file_link", "created": 1649752922, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfNzhlbE9MUGNYbkJzMkRLSWdEcnhvY3FH00DK5jBVaH"}, "emitted_at": 1697627273835} -{"stream": "checkout_sessions", "data": {"id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "object": "checkout.session", "after_expiration": null, "allow_promotion_codes": null, "amount_subtotal": 3400, "amount_total": 3400, "automatic_tax": {"enabled": false, "status": null}, "billing_address_collection": null, "cancel_url": null, "client_reference_id": null, "client_secret": null, "consent": null, "consent_collection": null, "created": 1697627124, "currency": "usd", "currency_conversion": null, "custom_fields": [], "custom_text": {"after_submit": null,"shipping_address": null, "submit": null, "terms_of_service_acceptance": null}, "customer": null, "customer_creation": "always", "customer_details": null, "customer_email": null, "expires_at": 1697713523, "invoice": null, "invoice_creation": {"enabled": false, "invoice_data": {"account_tax_ids": null, "custom_fields": null, "description": null, "footer": null, "metadata": {}, "rendering_options": null}}, "livemode": false, "locale": null, "metadata": {}, "mode": "payment", "payment_intent": "pi_3O2XZ1EcXtiJtvvh0zWGn33E", "payment_link": null, "payment_method_collection": "always", "payment_method_configuration_details": {"id": "pmc_1MC0oMEcXtiJtvvhmhbSUwTJ", "parent": null}, "payment_method_options": {"us_bank_account": {"financial_connections": {"permissions": ["payment_method"], "prefetch": []}, "verification_method": "automatic"}, "wechat_pay": {"app_id": null, "client": "web"}}, "payment_method_types": ["card", "alipay", "klarna", "link", "us_bank_account", "wechat_pay", "cashapp"], "payment_status": "unpaid", "phone_number_collection": {"enabled": false}, "recovered_from": null, "setup_intent": null, "shipping_address_collection": null, "shipping_cost": null, "shipping_details": null, "shipping_options": [], "status": "expired", "submit_type": null, "subscription": null, "success_url": "https://example.com/success", "total_details": {"amount_discount": 0, "amount_shipping": 0, "amount_tax": 0}, "ui_mode": "hosted", "url": null, "updated": 1697627124}, "emitted_at": 1697627275062} +{"stream": "checkout_sessions", "data": {"id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "object": "checkout.session", "after_expiration": null, "allow_promotion_codes": null, "amount_subtotal": 3400, "amount_total": 3400, "automatic_tax": {"enabled": false, "liability": null, "status": null}, "billing_address_collection": null, "cancel_url": null, "client_reference_id": null, "client_secret": null, "consent": null, "consent_collection": null, "created": 1697627124, "currency": "usd", "currency_conversion": null, "custom_fields": [], "custom_text": {"after_submit": null,"shipping_address": null, "submit": null, "terms_of_service_acceptance": null}, "customer": null, "customer_creation": "always", "customer_details": null, "customer_email": null, "expires_at": 1697713523, "invoice": null, "invoice_creation": {"enabled": false, "invoice_data": {"account_tax_ids": null, "custom_fields": null, "description": null, "footer": null, "issuer": null, "metadata": {}, "rendering_options": null}}, "livemode": false, "locale": null, "metadata": {}, "mode": "payment", "payment_intent": "pi_3O2XZ1EcXtiJtvvh0zWGn33E", "payment_link": null, "payment_method_collection": "always", "payment_method_configuration_details": {"id": "pmc_1MC0oMEcXtiJtvvhmhbSUwTJ", "parent": null}, "payment_method_options": {"us_bank_account": {"financial_connections": {"permissions": ["payment_method"], "prefetch": []}, "verification_method": "automatic"}, "wechat_pay": {"app_id": null, "client": "web"}}, "payment_method_types": ["card", "alipay", "klarna", "link", "us_bank_account", "wechat_pay", "cashapp"], "payment_status": "unpaid", "phone_number_collection": {"enabled": false}, "recovered_from": null, "setup_intent": null, "shipping_address_collection": null, "shipping_cost": null, "shipping_details": null, "shipping_options": [], "status": "expired", "submit_type": null, "subscription": null, "success_url": "https://example.com/success", "total_details": {"amount_discount": 0, "amount_shipping": 0, "amount_tax": 0}, "ui_mode": "hosted", "url": null, "updated": 1697627124}, "emitted_at": 1697627275062} {"stream": "credit_notes", "data": {"id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", "object": "credit_note", "amount": 8400, "amount_shipping": 0, "created": 1686158100, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "customer_balance_transaction": null, "discount_amount": "0", "discount_amounts": [], "effective_at": 1686158100, "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "lines": {"object": "list", "data": [{"id": "cnli_1NGPwmEcXtiJtvvhcL7yEIBJ", "object": "credit_note_line_item", "amount": 8400, "amount_excluding_tax": 8400, "description": "a box of parsnips", "discount_amount": 0, "discount_amounts": [], "invoice_line_item": "il_1K9GKLEcXtiJtvvhhHaYMebN", "livemode": false, "quantity": 1, "tax_amounts": [], "tax_rates": [], "type": "invoice_line_item", "unit_amount": 8400, "unit_amount_decimal": 8400.0, "unit_amount_excluding_tax": 8400.0}], "has_more": false, "url": "/v1/credit_notes/cn_1NGPwmEcXtiJtvvhNXwHpgJF/lines"}, "livemode": false, "memo": null, "metadata": {}, "number": "CA35DF83-0001-CN-01", "out_of_band_amount": null, "pdf": "https://pay.stripe.com/credit_notes/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9PMlV3dFlJelh4NHM1R0VIWnhMR3RjWUtlejFlRWtILDg4MTY4MDc20200Sa50llWu/pdf?s=ap", "reason": null, "refund": null, "shipping_cost": null, "status": "issued", "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax_amounts": [], "total": 8400, "total_excluding_tax": 8400, "type": "pre_payment", "voided_at": null, "updated": 1686158100}, "emitted_at": 1697627276386} {"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null, "updated": 1646998902}, "emitted_at": 1697627278433} {"stream": "customers", "data": {"id": "cus_Kou8knsO3qQOwU", "object": "customer", "address": null, "balance": 0, "created": 1640123795, "currency": "usd", "default_source": "src_1MSID8EcXtiJtvvhxIT9lXRy", "delinquent": false, "description": null, "discount": null, "email": "edward.gao+stripe-test-customer-1@airbyte.io", "invoice_prefix": "CA35DF83", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "edgao-test-customer-1", "next_invoice_sequence": 2, "phone": null, "preferred_locales": [], "shipping": null, "tax_exempt": "none", "test_clock": null, "updated": 1640123795}, "emitted_at": 1697627278435} @@ -47,7 +47,7 @@ {"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "features": [], "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1696839789, "url": null}, "emitted_at": 1697627307877} {"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "features": [], "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1696839225, "url": null}, "emitted_at": 1697627307879} {"stream": "subscriptions", "data": {"id": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true, "liability": {"type": "self"}}, "billing_cycle_anchor": 1697550676.0, "billing_cycle_anchor_config": null, "billing_thresholds": null, "cancel_at": null, "cancel_at_period_end": false, "canceled_at": 1697550676.0, "cancellation_details": {"comment": null, "feedback": null, "reason": "cancellation_requested"}, "collection_method": "charge_automatically", "created": 1697550676, "currency": "usd", "current_period_end": 1705499476.0, "current_period_start": 1702821076, "customer": "cus_NGoTFiJFVbSsvZ", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": 1705329724.0, "invoice_settings": {"issuer": {"type": "self"}}, "items": {"object": "list", "data": [{"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1O2Dg0EcXtiJtvvhz7Q4zS0n"}, "latest_invoice": "in_1OOKkUEcXtiJtvvheUUavyuB", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": null}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "start_date": 1697550676, "status": "canceled", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null, "updated": 1697550676}, "emitted_at": 1705636378387} -{"stream":"subscription_schedule","data":{"id":"sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP","object":"subscription_schedule","application":null,"canceled_at":"1705329724","completed_at":null,"created":1697550676,"current_phase":null,"customer":"cus_NGoTFiJFVbSsvZ","default_settings":{"application_fee_percent":null,"automatic_tax":{"enabled":false},"billing_cycle_anchor":"automatic","billing_thresholds":null,"collection_method":"charge_automatically","default_payment_method":null,"default_source":null,"description":"Test Test","invoice_settings":"{'days_until_due': None}","on_behalf_of":null,"transfer_data":null},"end_behavior":"cancel","livemode":false,"metadata":{},"phases":[{"add_invoice_items":[],"application_fee_percent":null,"automatic_tax":{"enabled":true},"billing_cycle_anchor":null,"billing_thresholds":null,"collection_method":"charge_automatically","coupon":null,"currency":"usd","default_payment_method":null,"default_tax_rates":[],"description":"Test Test","end_date":1705499476,"invoice_settings":"{'days_until_due': None}","items":[{"billing_thresholds":null,"metadata":{},"plan":"price_1MSHZoEcXtiJtvvh6O8TYD8T","price":"price_1MSHZoEcXtiJtvvh6O8TYD8T","quantity":1,"tax_rates":[]}],"metadata":{},"on_behalf_of":null,"proration_behavior":"create_prorations","start_date":1697550676,"transfer_data":null,"trial_end":null}],"released_at":null,"released_subscription":null,"renewal_interval":null,"status":"canceled","subscription":"sub_1O2Dg0EcXtiJtvvhz7Q4zS0n","test_clock":null,"updated":1697550676},"emitted_at":1705636378620} +{"stream":"subscription_schedule","data":{"id":"sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP","object":"subscription_schedule","application":null,"canceled_at":"1705329724","completed_at":null,"created":1697550676,"current_phase":null,"customer":"cus_NGoTFiJFVbSsvZ","default_settings":{"application_fee_percent":null,"automatic_tax":{"enabled":false, "liability": null},"billing_cycle_anchor":"automatic","billing_thresholds":null,"collection_method":"charge_automatically","default_payment_method":null,"default_source":null,"description":"Test Test","invoice_settings":"{'days_until_due': None, 'issuer': {'type': 'self'}}","on_behalf_of":null,"transfer_data":null},"end_behavior":"cancel","livemode":false,"metadata":{},"phases":[{"add_invoice_items":[],"application_fee_percent":null,"automatic_tax":{"enabled":true, "liability": {"type": "self"}},"billing_cycle_anchor":null,"billing_thresholds":null,"collection_method":"charge_automatically","coupon":null,"currency":"usd","default_payment_method":null,"default_tax_rates":[],"description":"Test Test","end_date":1705499476,"invoice_settings":"{'days_until_due': None, 'issuer': None}","items":[{"billing_thresholds":null,"metadata":{},"plan":"price_1MSHZoEcXtiJtvvh6O8TYD8T","price":"price_1MSHZoEcXtiJtvvh6O8TYD8T","quantity":1,"tax_rates":[]}],"metadata":{},"on_behalf_of":null,"proration_behavior":"create_prorations","start_date":1697550676,"transfer_data":null,"trial_end":null}],"released_at":null,"released_subscription":null,"renewal_interval":null,"status":"canceled","subscription":"sub_1O2Dg0EcXtiJtvvhz7Q4zS0n","test_clock":null,"updated":1697550676},"emitted_at":1705636378620} {"stream": "transfers", "data": {"id": "tr_1NH18zEcXtiJtvvhnd827cNO", "object": "transfer", "amount": 10000, "amount_reversed": 0, "balance_transaction": "txn_1NH190EcXtiJtvvhBO3PeR7p", "created": 1686301085, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NH18zEYmRTj5on1GkCCsqLK", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/transfers/tr_1NH18zEcXtiJtvvhnd827cNO/reversals"}, "reversed": false, "source_transaction": null, "source_type": "card", "transfer_group": null, "updated": 1686301085}, "emitted_at": 1697627313262} {"stream": "transfers", "data": {"id": "tr_1NGoaCEcXtiJtvvhjmHtOGOm", "object": "transfer", "amount": 100, "amount_reversed": 100, "balance_transaction": "txn_1NGoaDEcXtiJtvvhsZrNMsdJ", "created": 1686252800, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NGoaCEYmRTj5on1LAlAIG3a", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [{"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}], "has_more": false, "total_count": 1.0, "url": "/v1/transfers/tr_1NGoaCEcXtiJtvvhjmHtOGOm/reversals"}, "reversed": true, "source_transaction": null, "source_type": "card", "transfer_group": "ORDER10", "updated": 1686252800}, "emitted_at": 1697627313264} {"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "destination_details": {"card": {"reference": "5871771120000631", "reference_status": "available", "reference_type": "acquirer_reference_number", "type": "refund"}, "type": "card"}, "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1701882752716} diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index cfedccfdf01976..70ecc07cd95de4 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 5.2.0 + dockerImageTag: 5.2.1 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index b059da570f3212..c1b020664edb9e 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk==0.58.9", "stripe==2.56.0", "pendulum==2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk==0.59.1", "stripe==2.56.0", "pendulum==2.1.2"] # we set `requests-mock~=1.11.0` to ensure concurrency is supported TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock~=1.11.0", "requests_mock~=1.8", "freezegun==1.2.2"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json index b1a68dbcb95d04..36dc095652aa9e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json @@ -6,6 +6,24 @@ "business_profile": { "type": ["null", "object"], "properties": { + "annual_revenue": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "fiscal_year_end": { + "type": ["null", "string"] + } + } + }, + "estimated_worker_count": { + "type": ["null", "integer"] + }, "mcc": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json index b331e12c64270a..3aec7668d5ae44 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json @@ -25,6 +25,13 @@ "type": ["null", "object"], "properties": { "enabled": { "type": ["null", "boolean"] }, + "liability": { + "type": ["null", "object"], + "properties": { + "account": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] } + } + }, "status": { "type": ["null", "string"] } } }, @@ -369,6 +376,17 @@ "footer": { "type": ["null", "string"] }, + "issuer": { + "type": ["null", "object"], + "properties": { + "account": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, "metadata": { "type": ["null", "object"] }, diff --git a/airbyte-integrations/connectors/source-typeform/metadata.yaml b/airbyte-integrations/connectors/source-typeform/metadata.yaml index f8996479b0112d..5dd576c463f1f5 100644 --- a/airbyte-integrations/connectors/source-typeform/metadata.yaml +++ b/airbyte-integrations/connectors/source-typeform/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7eff203-90bf-43e5-a240-19ea3056c474 - dockerImageTag: 1.2.3 + dockerImageTag: 1.2.4 dockerRepository: airbyte/source-typeform documentationUrl: https://docs.airbyte.com/integrations/sources/typeform githubIssueLabel: source-typeform diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml b/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml index 3fa818405d3fa3..6fd25154e5e632 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml @@ -125,7 +125,7 @@ definitions: pagination_strategy: type: CursorPagination cursor_value: "{{ last_records[-1]['token'] }}" - stop_condition: "{{ not response['total_items'] }}" + stop_condition: "{{ not last_records }}" page_size: 1000 partition_router: $ref: "#/definitions/form_id_partition_router" diff --git a/airbyte-lib/README.md b/airbyte-lib/README.md index b30ced523d93ee..c65e5fba04fc00 100644 --- a/airbyte-lib/README.md +++ b/airbyte-lib/README.md @@ -9,6 +9,17 @@ airbyte-lib is a library that allows to run Airbyte syncs embedded into any Pyth * For examples, check out the `examples` folder. They can be run via `poetry run python examples/` * Unit tests and type checks can be run via `poetry run pytest` +## Release + +* In your PR: + * Bump the version in `pyproject.toml` + * Add a changelog entry to the table below +* Once the PR is merged, go to Github and trigger the `Publish AirbyteLib Manually` workflow. This will publish the new version to PyPI. + +### Versioning + +Versioning follows [Semantic Versioning](https://semver.org/). For new features, bump the minor version. For bug fixes, bump the patch version. For pre-releases, append `dev.N` to the version. For example, `0.1.0dev.1` is the first pre-release of the `0.1.0` version. + ## Documentation Regular documentation lives in the `/docs` folder. Based on the doc strings of public methods, we generate API documentation using [pdoc](https://pdoc.dev). To generate the documentation, run `poetry run generate-docs`. The documentation will be generated in the `docs/generate` folder. This needs to be done manually when changing the public interface of the library. @@ -24,3 +35,11 @@ airbyte-lib-validate-source —connector-dir . -—sample-config secrets/config. ``` The script will install the python package in the provided directory, and run the connector against the provided config. The config should be a valid JSON file, with the same structure as the one that would be provided to the connector in Airbyte. The script will exit with a non-zero exit code if the connector fails to run. + +For a more lightweight check, the `--validate-install-only` flag can be used. This will only check that the connector can be installed and returns a spec, no sample config required. + +## Changelog + +| Version | PR | Description | +| ----------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 0.1.0dev.2 | [#34111](https://github.com/airbytehq/airbyte/pull/34111) | Initial publish - add publish workflow | diff --git a/airbyte-lib/airbyte_lib/__init__.py b/airbyte-lib/airbyte_lib/__init__.py index 895849f19771ab..86ffb5e0fa4016 100644 --- a/airbyte-lib/airbyte_lib/__init__.py +++ b/airbyte-lib/airbyte_lib/__init__.py @@ -1,17 +1,21 @@ """AirbyteLib brings Airbyte ELT to every Python developer.""" +from __future__ import annotations from airbyte_lib._factories.cache_factories import get_default_cache, new_local_cache from airbyte_lib._factories.connector_factories import get_connector +from airbyte_lib.caches import DuckDBCache, DuckDBCacheConfig from airbyte_lib.datasets import CachedDataset from airbyte_lib.results import ReadResult from airbyte_lib.source import Source __all__ = [ + "CachedDataset", + "DuckDBCache", + "DuckDBCacheConfig", "get_connector", "get_default_cache", "new_local_cache", - "CachedDataset", "ReadResult", "Source", ] diff --git a/airbyte-lib/airbyte_lib/_executor.py b/airbyte-lib/airbyte_lib/_executor.py index 1a816cc46848f7..20899f892006ca 100644 --- a/airbyte-lib/airbyte_lib/_executor.py +++ b/airbyte-lib/airbyte_lib/_executor.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import IO, TYPE_CHECKING, Any, NoReturn +from airbyte_lib import exceptions as exc from airbyte_lib.telemetry import SourceTelemetryInfo, SourceType @@ -71,7 +72,13 @@ def _stream_from_file(file: IO[str]) -> Generator[str, Any, None]: yield line if process.stdout is None: - raise Exception("Failed to start subprocess") + raise exc.AirbyteSubprocessError( + message="Subprocess did not return a stdout stream.", + context={ + "args": args, + "returncode": process.returncode, + }, + ) try: yield _stream_from_file(process.stdout) finally: @@ -94,7 +101,7 @@ def _stream_from_file(file: IO[str]) -> Generator[str, Any, None]: # If the exit code is not 0 or -15 (SIGTERM), raise an exception if exit_code not in (0, -15): - raise Exception(f"Process exited with code {exit_code}") + raise exc.AirbyteSubprocessFailedError(exit_code=exit_code) class VenvExecutor(Executor): @@ -123,7 +130,9 @@ def _get_connector_path(self) -> Path: def _run_subprocess_and_raise_on_failure(self, args: list[str]) -> None: result = subprocess.run(args, check=False) if result.returncode != 0: - raise Exception(f"Install process exited with code {result.returncode}") + raise exc.AirbyteConnectorInstallationError from exc.AirbyteSubprocessFailedError( + exit_code=result.returncode + ) def uninstall(self) -> None: venv_name = self._get_venv_name() @@ -171,18 +180,27 @@ def ensure_installation( venv_path = Path(venv_name) if not venv_path.exists(): if not self.install_if_missing: - raise Exception( - f"Connector {self.metadata.name} is not available - " - f"venv {venv_name} does not exist" + raise exc.AirbyteConnectorNotFoundError( + message="Connector not available and venv does not exist.", + guidance=( + "Please ensure the connector is pre-installed or consider enabling " + "`install_if_missing=True`." + ), + context={ + "connector_name": self.metadata.name, + "venv_name": venv_name, + }, ) self.install() connector_path = self._get_connector_path() if not connector_path.exists(): - raise FileNotFoundError( - f"Could not find connector '{self.metadata.name}' in venv '{venv_name}' with " - f"connector path '{connector_path}'.", - ) + raise exc.AirbyteConnectorNotFoundError( + connector_name=self.metadata.name, + context={ + "venv_name": venv_name, + }, + ) from FileNotFoundError(connector_path) if self.enforce_version: installed_version = self._get_installed_version() @@ -193,9 +211,14 @@ def ensure_installation( # Check the version again version_after_install = self._get_installed_version() if version_after_install != self.target_version: - raise Exception( - f"Failed to install connector {self.metadata.name} version " - f"{self.target_version}. Installed version is {version_after_install}", + raise exc.AirbyteConnectorInstallationError( + connector_name=self.metadata.name, + context={ + "venv_name": venv_name, + "target_version": self.target_version, + "installed_version": installed_version, + "version_after_install": version_after_install, + }, ) def execute(self, args: list[str]) -> Iterator[str]: @@ -213,17 +236,20 @@ def ensure_installation(self) -> None: try: self.execute(["spec"]) except Exception as e: - raise Exception( - f"Connector {self.metadata.name} is not available - executing it failed" + raise exc.AirbyteConnectorNotFoundError( + connector_name=self.metadata.name, ) from e def install(self) -> NoReturn: - raise Exception(f"Connector {self.metadata.name} is not available - cannot install it") + raise exc.AirbyteConnectorInstallationError( + message="Connector cannot be installed because it is not managed by airbyte-lib.", + connector_name=self.metadata.name, + ) def uninstall(self) -> NoReturn: - raise Exception( - f"Connector {self.metadata.name} is installed manually and not managed by airbyte-lib -" - " please remove it manually" + raise exc.AirbyteConnectorInstallationError( + message="Connector cannot be uninstalled because it is not managed by airbyte-lib.", + connector_name=self.metadata.name, ) def execute(self, args: list[str]) -> Iterator[str]: diff --git a/airbyte-lib/airbyte_lib/_factories/cache_factories.py b/airbyte-lib/airbyte_lib/_factories/cache_factories.py index 5a95dce2db7b43..82ad3241920ccb 100644 --- a/airbyte-lib/airbyte_lib/_factories/cache_factories.py +++ b/airbyte-lib/airbyte_lib/_factories/cache_factories.py @@ -5,6 +5,7 @@ import ulid +from airbyte_lib import exceptions as exc from airbyte_lib.caches.duckdb import DuckDBCache, DuckDBCacheConfig @@ -38,12 +39,15 @@ def new_local_cache( """ if cache_name: if " " in cache_name: - raise ValueError(f"Cache name '{cache_name}' cannot contain spaces") + raise exc.AirbyteLibInputError( + message="Cache name cannot contain spaces.", + input_value=cache_name, + ) if not cache_name.replace("_", "").isalnum(): - raise ValueError( - f"Cache name '{cache_name}' can only contain alphanumeric " - "characters and underscores." + raise exc.AirbyteLibInputError( + message="Cache name can only contain alphanumeric characters and underscores.", + input_value=cache_name, ) cache_name = cache_name or str(ulid.ULID()) diff --git a/airbyte-lib/airbyte_lib/_factories/connector_factories.py b/airbyte-lib/airbyte_lib/_factories/connector_factories.py index 347710f20824a2..4dbe8c6f41f06b 100644 --- a/airbyte-lib/airbyte_lib/_factories/connector_factories.py +++ b/airbyte-lib/airbyte_lib/_factories/connector_factories.py @@ -4,6 +4,7 @@ from typing import Any from airbyte_lib._executor import Executor, PathExecutor, VenvExecutor +from airbyte_lib.exceptions import AirbyteLibInputError from airbyte_lib.registry import get_connector_metadata from airbyte_lib.source import Source @@ -37,9 +38,13 @@ def get_connector( metadata = get_connector_metadata(name) if use_local_install: if pip_url: - raise ValueError("Param 'pip_url' is not supported when 'use_local_install' is True") + raise AirbyteLibInputError( + message="Param 'pip_url' is not supported when 'use_local_install' is True." + ) if version: - raise ValueError("Param 'version' is not supported when 'use_local_install' is True") + raise AirbyteLibInputError( + message="Param 'version' is not supported when 'use_local_install' is True." + ) executor: Executor = PathExecutor( metadata=metadata, target_version=version, diff --git a/airbyte-lib/airbyte_lib/_file_writers/__init__.py b/airbyte-lib/airbyte_lib/_file_writers/__init__.py index 007dde83243455..aae8c474ca97f6 100644 --- a/airbyte-lib/airbyte_lib/_file_writers/__init__.py +++ b/airbyte-lib/airbyte_lib/_file_writers/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from .base import FileWriterBase, FileWriterBatchHandle, FileWriterConfigBase from .parquet import ParquetWriter, ParquetWriterConfig diff --git a/airbyte-lib/airbyte_lib/_processors.py b/airbyte-lib/airbyte_lib/_processors.py index f0f94c30c512df..ee94181660be66 100644 --- a/airbyte-lib/airbyte_lib/_processors.py +++ b/airbyte-lib/airbyte_lib/_processors.py @@ -30,6 +30,7 @@ Type, ) +from airbyte_lib import exceptions as exc from airbyte_lib._util import protocol_util # Internal utility functions @@ -99,6 +100,11 @@ def register_source( _ = source_name self.source_catalog = source_catalog + @property + def _streams_with_data(self) -> set[str]: + """Return a list of known streams.""" + return self._pending_batches.keys() | self._finalized_batches.keys() + @final def process_stdin( self, @@ -166,7 +172,12 @@ def process_airbyte_messages( pass else: - raise ValueError(f"Unexpected message type: {message.type}") + raise exc.AirbyteConnectorError( + message="Unexpected message type.", + context={ + "message_type": message.type, + }, + ) # We are at the end of the stream. Process whatever else is queued. for stream_name, batch in stream_batches.items(): diff --git a/airbyte-lib/airbyte_lib/_util/protocol_util.py b/airbyte-lib/airbyte_lib/_util/protocol_util.py index 58ada9f5435b26..2ddaa1346e307e 100644 --- a/airbyte-lib/airbyte_lib/_util/protocol_util.py +++ b/airbyte-lib/airbyte_lib/_util/protocol_util.py @@ -12,6 +12,8 @@ Type, ) +from airbyte_lib import exceptions as exc + if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -24,7 +26,7 @@ def airbyte_messages_to_record_dicts( yield from ( cast(dict[str, Any], airbyte_message_to_record_dict(message)) for message in messages - if message is not None + if message is not None and message.type == Type.RECORD ) @@ -66,6 +68,10 @@ def get_primary_keys_from_stream( None, ) if stream is None: - raise ValueError(f"Stream {stream_name} not found in catalog.") + raise exc.AirbyteStreamNotFoundError( + stream_name=stream_name, + connector_name=configured_catalog.connection.configuration["name"], + available_streams=[stream.stream.name for stream in configured_catalog.streams], + ) return set(stream.stream.source_defined_primary_key or []) diff --git a/airbyte-lib/airbyte_lib/caches/__init__.py b/airbyte-lib/airbyte_lib/caches/__init__.py index d14980332458a6..3cb5c31cf11920 100644 --- a/airbyte-lib/airbyte_lib/caches/__init__.py +++ b/airbyte-lib/airbyte_lib/caches/__init__.py @@ -1,4 +1,5 @@ """Base module for all caches.""" +from __future__ import annotations from airbyte_lib.caches.base import SQLCacheBase from airbyte_lib.caches.duckdb import DuckDBCache, DuckDBCacheConfig diff --git a/airbyte-lib/airbyte_lib/caches/base.py b/airbyte-lib/airbyte_lib/caches/base.py index 0e3ad837b194f5..23df721400b856 100644 --- a/airbyte-lib/airbyte_lib/caches/base.py +++ b/airbyte-lib/airbyte_lib/caches/base.py @@ -5,7 +5,6 @@ import abc import enum -from collections.abc import Generator, Iterator, Mapping from contextlib import contextmanager from functools import cached_property from typing import TYPE_CHECKING, Any, cast, final @@ -19,13 +18,16 @@ from sqlalchemy.pool import StaticPool from sqlalchemy.sql.elements import TextClause +from airbyte_lib import exceptions as exc from airbyte_lib._file_writers.base import FileWriterBase, FileWriterBatchHandle from airbyte_lib._processors import BatchHandle, RecordProcessor from airbyte_lib.config import CacheConfigBase +from airbyte_lib.datasets._sql import CachedDataset from airbyte_lib.types import SQLTypeConverter if TYPE_CHECKING: + from collections.abc import Generator, Iterator from pathlib import Path from sqlalchemy.engine import Connection, Engine @@ -118,6 +120,15 @@ def __init__( self.file_writer = file_writer or self.file_writer_class(config) self.type_converter = self.type_converter_class() + def __getitem__(self, stream: str) -> DatasetBase: + return self.streams[stream] + + def __contains__(self, stream: str) -> bool: + return stream in self._streams_with_data + + def __iter__(self) -> Iterator[str]: + return iter(self._streams_with_data) + # Public interface: def get_sql_alchemy_url(self) -> str: @@ -211,28 +222,22 @@ def get_sql_table( @property def streams( self, - ) -> dict[str, DatasetBase]: + ) -> dict[str, CachedDataset]: """Return a temporary table name.""" - # TODO: Add support for streams map, based on the cached catalog. - raise NotImplementedError("Streams map is not yet supported.") + result = {} + for stream_name in self._streams_with_data: + result[stream_name] = CachedDataset(self, stream_name) + + return result # Read methods: def get_records( self, stream_name: str, - ) -> Iterator[Mapping[str, Any]]: - """Uses SQLAlchemy to select all rows from the table. - - # TODO: Refactor to return a LazyDataset here. - """ - table_ref = self.get_sql_table(stream_name) - stmt = table_ref.select() - with self.get_sql_connection() as conn: - for row in conn.execute(stmt): - # Access to private member required because SQLAlchemy doesn't expose a public API. - # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.engine.row.RowMapping.html - yield cast(Mapping[str, Any], row._mapping) # noqa: SLF001 + ) -> CachedDataset: + """Uses SQLAlchemy to select all rows from the table.""" + return CachedDataset(self, stream_name) def get_pandas_dataframe( self, @@ -370,8 +375,11 @@ def _ensure_compatible_table_schema( missing_columns: set[str] = set(stream_column_names) - set(table_column_names) if missing_columns: if raise_on_error: - raise RuntimeError( - f"Table {table_name} is missing columns: {missing_columns}", + raise exc.AirbyteLibCacheTableValidationError( + violation="Cache table is missing expected columns.", + context={ + "missing_columns": missing_columns, + }, ) return False # Some columns are missing. @@ -436,16 +444,25 @@ def _get_stream_config( ) -> ConfiguredAirbyteStream: """Return the column definitions for the given stream.""" if not self.source_catalog: - raise RuntimeError("Cannot get stream JSON schema without a catalog.") + raise exc.AirbyteLibInternalError( + message="Cannot get stream JSON schema without a catalog.", + ) matching_streams: list[ConfiguredAirbyteStream] = [ stream for stream in self.source_catalog.streams if stream.stream.name == stream_name ] if not matching_streams: - raise RuntimeError(f"Stream '{stream_name}' not found in catalog.") + raise exc.AirbyteStreamNotFoundError( + stream_name=stream_name, + ) if len(matching_streams) > 1: - raise RuntimeError(f"Multiple streams found with name '{stream_name}'.") + raise exc.AirbyteLibInternalError( + message="Multiple streams found with same name.", + context={ + "stream_name": stream_name, + }, + ) return matching_streams[0] @@ -521,12 +538,12 @@ def _finalize_batches(self, stream_name: str) -> dict[str, BatchHandle]: raise_on_error=True, ) + temp_table_name = self._write_files_to_new_table( + files, + stream_name, + max_batch_id, + ) try: - temp_table_name = self._write_files_to_new_table( - files, - stream_name, - max_batch_id, - ) self._write_temp_table_to_final_table( stream_name, temp_table_name, @@ -588,7 +605,12 @@ def _write_files_to_new_table( # Pandas will auto-create the table if it doesn't exist, which we don't want. if not self._table_exists(temp_table_name): - raise RuntimeError(f"Table {temp_table_name} does not exist after creation.") + raise exc.AirbyteLibInternalError( + message="Table does not exist after creation.", + context={ + "temp_table_name": temp_table_name, + }, + ) dataframe.to_sql( temp_table_name, @@ -681,9 +703,9 @@ def _swap_temp_table_with_final_table( Databases that do not support this syntax can override this method. """ if final_table_name is None: - raise ValueError("Arg 'final_table_name' cannot be None.") + raise exc.AirbyteLibInternalError(message="Arg 'final_table_name' cannot be None.") if temp_table_name is None: - raise ValueError("Arg 'temp_table_name' cannot be None.") + raise exc.AirbyteLibInternalError(message="Arg 'temp_table_name' cannot be None.") _ = stream_name deletion_name = f"{final_table_name}_deleteme" diff --git a/airbyte-lib/airbyte_lib/caches/duckdb.py b/airbyte-lib/airbyte_lib/caches/duckdb.py index 0d6ba6efe38a9e..ac70bcf8648ad0 100644 --- a/airbyte-lib/airbyte_lib/caches/duckdb.py +++ b/airbyte-lib/airbyte_lib/caches/duckdb.py @@ -9,6 +9,7 @@ from overrides import overrides +from airbyte_lib import exceptions as exc from airbyte_lib._file_writers import ParquetWriter, ParquetWriterConfig from airbyte_lib.caches.base import SQLCacheBase, SQLCacheConfigBase from airbyte_lib.telemetry import CacheTelemetryInfo @@ -93,9 +94,11 @@ def _merge_temp_table_to_final_table( Databases that do not support this syntax can override this method. """ if not self._get_primary_keys(stream_name): - raise RuntimeError( - f"Primary keys not found for stream {stream_name}. " - "Cannot run merge updates without primary keys." + raise exc.AirbyteLibInternalError( + message="Primary keys not found. Cannot run merge updates without primary keys.", + context={ + "stream_name": stream_name, + }, ) _ = stream_name @@ -135,10 +138,14 @@ def _ensure_compatible_table_schema( table_pk_cols = table.primary_key.columns.keys() if set(pk_cols) != set(table_pk_cols): if raise_on_error: - raise RuntimeError( - f"Primary keys do not match for table {table_name}. " - f"Expected: {pk_cols}. " - f"Found: {table_pk_cols}.", + raise exc.AirbyteLibCacheTableValidationError( + violation="Primary keys do not match.", + context={ + "stream_name": stream_name, + "table_name": table_name, + "expected": pk_cols, + "found": table_pk_cols, + }, ) return False diff --git a/airbyte-lib/airbyte_lib/caches/postgres.py b/airbyte-lib/airbyte_lib/caches/postgres.py index 72fe8291bf5431..5c7df3a898adf5 100644 --- a/airbyte-lib/airbyte_lib/caches/postgres.py +++ b/airbyte-lib/airbyte_lib/caches/postgres.py @@ -28,9 +28,7 @@ class PostgresCacheConfig(SQLCacheConfigBase, ParquetWriterConfig): @overrides def get_sql_alchemy_url(self) -> str: """Return the SQLAlchemy URL to use.""" - return ( - f"postgresql://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" - ) + return f"postgresql+psycopg2://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" def get_database_name(self) -> str: """Return the name of the database.""" diff --git a/airbyte-lib/airbyte_lib/datasets/__init__.py b/airbyte-lib/airbyte_lib/datasets/__init__.py index 862eee3e8baf1f..bfd4f02ce319a7 100644 --- a/airbyte-lib/airbyte_lib/datasets/__init__.py +++ b/airbyte-lib/airbyte_lib/datasets/__init__.py @@ -1,10 +1,15 @@ +from __future__ import annotations + from airbyte_lib.datasets._base import DatasetBase -from airbyte_lib.datasets._cached import CachedDataset +from airbyte_lib.datasets._lazy import LazyDataset from airbyte_lib.datasets._map import DatasetMap +from airbyte_lib.datasets._sql import CachedDataset, SQLDataset __all__ = [ "CachedDataset", "DatasetBase", "DatasetMap", + "LazyDataset", + "SQLDataset", ] diff --git a/airbyte-lib/airbyte_lib/datasets/_base.py b/airbyte-lib/airbyte_lib/datasets/_base.py index b42e5258e5593f..f0fdfab52b9129 100644 --- a/airbyte-lib/airbyte_lib/datasets/_base.py +++ b/airbyte-lib/airbyte_lib/datasets/_base.py @@ -1,27 +1,26 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping from typing import Any, cast from pandas import DataFrame -from typing_extensions import Self -class DatasetBase(ABC, Iterator[Mapping[str, Any]]): +class DatasetBase(ABC): """Base implementation for all datasets.""" - def __iter__(self) -> Self: - """Return the iterator object (usually self).""" - return self - @abstractmethod - def __next__(self) -> Mapping[str, Any]: - """Return the next value from the iterator.""" + def __iter__(self) -> Iterator[Mapping[str, Any]]: + """Return the iterator of records.""" raise NotImplementedError def to_pandas(self) -> DataFrame: - """Return a pandas DataFrame representation of the dataset.""" + """Return a pandas DataFrame representation of the dataset. + + The base implementation simply passes the record iterator to Panda's DataFrame constructor. + """ # Technically, we return an iterator of Mapping objects. However, pandas # expects an iterator of dict objects. This cast is safe because we know # duck typing is correct for this use case. diff --git a/airbyte-lib/airbyte_lib/datasets/_cached.py b/airbyte-lib/airbyte_lib/datasets/_cached.py deleted file mode 100644 index 37aed0458312b9..00000000000000 --- a/airbyte-lib/airbyte_lib/datasets/_cached.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any - -from typing_extensions import Self - -from airbyte_lib.datasets._base import DatasetBase - - -if TYPE_CHECKING: - from pandas import DataFrame - from sqlalchemy import Table - - from airbyte_lib.caches import SQLCacheBase - - -class CachedDataset(DatasetBase): - def __init__(self, cache: "SQLCacheBase", stream: str) -> None: - self._cache = cache - self._stream = stream - self._iterator = iter(self._cache.get_records(self._stream)) - - def __iter__(self) -> Self: - return self - - def __next__(self) -> Mapping[str, Any]: - return next(self._iterator) - - def to_pandas(self) -> "DataFrame": - return self._cache.get_pandas_dataframe(self._stream) - - def to_sql_table(self) -> "Table": - return self._cache.get_sql_table(self._stream) diff --git a/airbyte-lib/airbyte_lib/datasets/_lazy.py b/airbyte-lib/airbyte_lib/datasets/_lazy.py index e08f844c407812..a8bb8173711de0 100644 --- a/airbyte-lib/airbyte_lib/datasets/_lazy.py +++ b/airbyte-lib/airbyte_lib/datasets/_lazy.py @@ -4,44 +4,26 @@ from typing import TYPE_CHECKING, Any from overrides import overrides -from typing_extensions import Self from airbyte_lib.datasets import DatasetBase if TYPE_CHECKING: - from collections.abc import Callable, Iterator + from collections.abc import Iterator, Mapping class LazyDataset(DatasetBase): - """A dataset that is loaded incrementally from a source or a SQL query. - - TODO: Test and debug this. It is not yet implemented anywhere in the codebase. - For now it servers as a placeholder. - """ + """A dataset that is loaded incrementally from a source or a SQL query.""" def __init__( self, - iterator: Iterator, - on_open: Callable | None = None, - on_close: Callable | None = None, + iterator: Iterator[Mapping[str, Any]], ) -> None: - self._iterator = iterator - self._on_open = on_open - self._on_close = on_close - raise NotImplementedError("This class is not implemented yet.") + self._iterator: Iterator[Mapping[str, Any]] = iterator @overrides - def __iter__(self) -> Self: - raise NotImplementedError("This class is not implemented yet.") - # Pseudocode: - # if self._on_open is not None: - # self._on_open() - - # yield from self._iterator - - # if self._on_close is not None: - # self._on_close() + def __iter__(self) -> Iterator[Mapping[str, Any]]: + return self._iterator - def __next__(self) -> dict[str, Any]: + def __next__(self) -> Mapping[str, Any]: return next(self._iterator) diff --git a/airbyte-lib/airbyte_lib/datasets/_map.py b/airbyte-lib/airbyte_lib/datasets/_map.py index 3881e1d33da841..42eaed88f0e3ef 100644 --- a/airbyte-lib/airbyte_lib/datasets/_map.py +++ b/airbyte-lib/airbyte_lib/datasets/_map.py @@ -5,10 +5,14 @@ TODO: This is a work in progress. It is not yet used by any other code. TODO: Implement before release, or delete. """ +from __future__ import annotations from collections.abc import Iterator, Mapping +from typing import TYPE_CHECKING -from airbyte_lib.datasets._base import DatasetBase + +if TYPE_CHECKING: + from airbyte_lib.datasets._base import DatasetBase class DatasetMap(Mapping): diff --git a/airbyte-lib/airbyte_lib/datasets/_sql.py b/airbyte-lib/airbyte_lib/datasets/_sql.py new file mode 100644 index 00000000000000..c6195dc6e28007 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_sql.py @@ -0,0 +1,119 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, cast + +from overrides import overrides +from sqlalchemy import and_, text + +from airbyte_lib.datasets._base import DatasetBase + + +if TYPE_CHECKING: + from collections.abc import Iterator + + from pandas import DataFrame + from sqlalchemy import Selectable, Table + from sqlalchemy.sql import ClauseElement + + from airbyte_lib.caches import SQLCacheBase + + +class SQLDataset(DatasetBase): + """A dataset that is loaded incrementally from a SQL query. + + The CachedDataset class is a subclass of this class, which simply passes a SELECT over the full + table as the query statement. + """ + + def __init__( + self, + cache: SQLCacheBase, + stream_name: str, + query_statement: Selectable, + ) -> None: + self._cache: SQLCacheBase = cache + self._stream_name: str = stream_name + self._query_statement: Selectable = query_statement + + @property + def stream_name(self) -> str: + return self._stream_name + + def __iter__(self) -> Iterator[Mapping[str, Any]]: + with self._cache.get_sql_connection() as conn: + for row in conn.execute(self._query_statement): + # Access to private member required because SQLAlchemy doesn't expose a public API. + # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.engine.row.RowMapping.html + yield cast(Mapping[str, Any], row._mapping) # noqa: SLF001 + + def to_pandas(self) -> DataFrame: + return self._cache.get_pandas_dataframe(self._stream_name) + + def with_filter(self, *filter_expressions: ClauseElement | str) -> SQLDataset: + """Filter the dataset by a set of column values. + + Filters can be specified as either a string or a SQLAlchemy expression. + + Filters are lazily applied to the dataset, so they can be chained together. For example: + + dataset.with_filter("id > 5").with_filter("id < 10") + + is equivalent to: + + dataset.with_filter("id > 5", "id < 10") + """ + # Convert all strings to TextClause objects. + filters: list[ClauseElement] = [ + text(expression) if isinstance(expression, str) else expression + for expression in filter_expressions + ] + filtered_select = self._query_statement.where(and_(*filters)) + return SQLDataset( + cache=self._cache, + stream_name=self._stream_name, + query_statement=filtered_select, + ) + + +class CachedDataset(SQLDataset): + """A dataset backed by a SQL table cache. + + Because this dataset includes all records from the underlying table, we also expose the + underlying table as a SQLAlchemy Table object. + """ + + def __init__(self, cache: SQLCacheBase, stream_name: str) -> None: + self._cache: SQLCacheBase = cache + self._stream_name: str = stream_name + self._query_statement: Selectable = self.to_sql_table().select() + + @overrides + def to_pandas(self) -> DataFrame: + return self._cache.get_pandas_dataframe(self._stream_name) + + def to_sql_table(self) -> Table: + return self._cache.get_sql_table(self._stream_name) + + def __eq__(self, value: object) -> bool: + """Return True if the value is a CachedDataset with the same cache and stream name. + + In the case of CachedDataset objects, we can simply compare the cache and stream name. + + Note that this equality check is only supported on CachedDataset objects and not for + the base SQLDataset implementation. This is because of the complexity and computational + cost of comparing two arbitrary SQL queries that could be bound to different variables, + as well as the chance that two queries can be syntactically equivalent without being + text-wise equivalent. + """ + if not isinstance(value, SQLDataset): + return False + + if self._cache is not value._cache: + return False + + if self._stream_name != value._stream_name: + return False + + return True diff --git a/airbyte-lib/airbyte_lib/exceptions.py b/airbyte-lib/airbyte_lib/exceptions.py new file mode 100644 index 00000000000000..3c6336d031033a --- /dev/null +++ b/airbyte-lib/airbyte_lib/exceptions.py @@ -0,0 +1,227 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""All exceptions used in the Airbyte Lib. + +This design is modeled after structlog's exceptions, in that we bias towards auto-generated +property prints rather than sentence-like string concatenation. + +E.g. Instead of this: +> Subprocess failed with exit code '1' + +We do this: +> Subprocess failed. (exit_code=1) + +The benefit of this approach is that we can easily support structured logging, and we can +easily add new properties to exceptions without having to update all the places where they +are raised. We can also support any arbitrary number of properties in exceptions, without spending +time on building sentence-like string constructions with optional inputs. + + +In addition, the following principles are applied for exception class design: + +- All exceptions inherit from a common base class. +- All exceptions have a message attribute. +- The first line of the docstring is used as the default message. +- The default message can be overridden by explicitly setting the message attribute. +- Exceptions may optionally have a guidance attribute. +- Exceptions may optionally have a help_url attribute. +- Rendering is automatically handled by the base class. +- Any helpful context not defined by the exception class can be passed in the `context` dict arg. +- Within reason, avoid sending PII to the exception constructor. +- Exceptions are dataclasses, so they can be instantiated with keyword arguments. +- Use the 'from' syntax to chain exceptions when it is helpful to do so. + E.g. `raise AirbyteConnectorNotFoundError(...) from FileNotFoundError(connector_path)` +- Any exception that adds a new property should also be decorated as `@dataclass`. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +NEW_ISSUE_URL = "https://github.com/airbytehq/airbyte/issues/new/choose" +DOCS_URL = "https://docs.airbyte.io/" + + +# Base error class + + +@dataclass +class AirbyteError(Exception): + """Base class for exceptions in Airbyte.""" + + guidance: str | None = None + help_url: str | None = None + log_text: str | list[str] | None = None + context: dict[str, Any] | None = None + message: str | None = None + + def get_message(self) -> str: + """Return the best description for the exception. + + We resolve the following in order: + 1. The message sent to the exception constructor (if provided). + 2. The first line of the class's docstring. + """ + if self.message: + return self.message + + return self.__doc__.split("\n")[0] if self.__doc__ else "" + + def __str__(self) -> str: + special_properties = ["message", "guidance", "help_url", "log_text"] + properties_str = ", ".join( + f"{k}={v!r}" + for k, v in self.__dict__.items() + if k not in special_properties and not k.startswith("_") and v is not None + ) + exception_str = f"{self.__class__.__name__}: {self.get_message()}." + if properties_str: + exception_str += f" ({properties_str})" + + if self.log_text: + if isinstance(self.log_text, list): + self.log_text = "\n".join(self.log_text) + + exception_str += f"\n\n Log output: {self.log_text}" + + if self.guidance: + exception_str += f"\n\n Suggestion: {self.guidance}" + + if self.help_url: + exception_str += f"\n\n More info: {self.help_url}" + + return exception_str + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + properties_str = ", ".join( + f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_") + ) + return f"{class_name}({properties_str})" + + +# AirbyteLib Internal Errors (these are probably bugs) + + +@dataclass +class AirbyteLibInternalError(AirbyteError): + """An internal error occurred in Airbyte Lib.""" + + guidance = "Please consider reporting this error to the Airbyte team." + help_url = NEW_ISSUE_URL + + +# AirbyteLib Input Errors (replaces ValueError for user input) + + +@dataclass +class AirbyteLibInputError(AirbyteError, ValueError): + """The input provided to AirbyteLib did not match expected validation rules. + + This inherits from ValueError so that it can be used as a drop-in replacement for + ValueError in the Airbyte Lib API. + """ + + # TODO: Consider adding a help_url that links to the auto-generated API reference. + + guidance = "Please check the provided value and try again." + input_value: str | None = None + + +# AirbyteLib Cache Errors + + +class AirbyteLibCacheError(AirbyteError): + """Error occurred while accessing the cache.""" + + +@dataclass +class AirbyteLibCacheTableValidationError(AirbyteLibCacheError): + """Cache table validation failed.""" + + violation: str | None = None + + +@dataclass +class AirbyteConnectorConfigurationMissingError(AirbyteLibCacheError): + """Connector is missing configuration.""" + + connector_name: str | None = None + + +# Subprocess Errors + + +@dataclass +class AirbyteSubprocessError(AirbyteError): + """Error when running subprocess.""" + + run_args: list[str] | None = None + + +@dataclass +class AirbyteSubprocessFailedError(AirbyteSubprocessError): + """Subprocess failed.""" + + exit_code: int | None = None + + +# Connector Registry Errors + + +class AirbyteConnectorRegistryError(AirbyteError): + """Error when accessing the connector registry.""" + + +# Connector Errors + + +@dataclass +class AirbyteConnectorError(AirbyteError): + """Error when running the connector.""" + + connector_name: str | None = None + + +class AirbyteConnectorNotFoundError(AirbyteConnectorError): + """Connector not found.""" + + +class AirbyteConnectorInstallationError(AirbyteConnectorError): + """Error when installing the connector.""" + + +class AirbyteConnectorReadError(AirbyteConnectorError): + """Error when reading from the connector.""" + + +class AirbyteNoDataFromConnectorError(AirbyteConnectorError): + """No data was provided from the connector.""" + + +class AirbyteConnectorMissingCatalogError(AirbyteConnectorError): + """Connector did not return a catalog.""" + + +class AirbyteConnectorMissingSpecError(AirbyteConnectorError): + """Connector did not return a spec.""" + + +class AirbyteConnectorCheckFailedError(AirbyteConnectorError): + """Connector did not return a spec.""" + + +@dataclass +class AirbyteConnectorFailedError(AirbyteConnectorError): + """Connector failed.""" + + exit_code: int | None = None + + +@dataclass +class AirbyteStreamNotFoundError(AirbyteConnectorError): + """Connector stream not found.""" + + stream_name: str | None = None + available_streams: list[str] | None = None diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index e0afdbaf2c3acb..bd030a867ff003 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -8,6 +8,7 @@ import requests +from airbyte_lib import exceptions as exc from airbyte_lib.version import get_version @@ -47,5 +48,11 @@ def get_connector_metadata(name: str) -> ConnectorMetadata: if not _cache: _update_cache() if not _cache or name not in _cache: - raise Exception(f"Connector {name} not found") + raise exc.AirbyteLibInputError( + message="Connector name not found in registry.", + guidance="Please double check the connector name.", + context={ + "connector_name": name, + }, + ) return _cache[name] diff --git a/airbyte-lib/airbyte_lib/results.py b/airbyte-lib/airbyte_lib/results.py index 3261416c130b79..81e67fad289d65 100644 --- a/airbyte-lib/airbyte_lib/results.py +++ b/airbyte-lib/airbyte_lib/results.py @@ -1,22 +1,43 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations -from sqlalchemy.engine import Engine +from typing import TYPE_CHECKING -from airbyte_lib.caches import SQLCacheBase from airbyte_lib.datasets import CachedDataset +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + + from sqlalchemy.engine import Engine + + from airbyte_lib.caches import SQLCacheBase + + class ReadResult: def __init__(self, processed_records: int, cache: SQLCacheBase) -> None: self.processed_records = processed_records self._cache = cache def __getitem__(self, stream: str) -> CachedDataset: + if stream not in self._cache: + raise KeyError(stream) + return CachedDataset(self._cache, stream) + def __contains__(self, stream: str) -> bool: + return stream in self._cache + + def __iter__(self) -> Iterator[str]: + return self._cache.__iter__() + def get_sql_engine(self) -> Engine: return self._cache.get_sql_engine() + @property + def streams(self) -> Mapping[str, CachedDataset]: + return self._cache.streams + @property def cache(self) -> SQLCacheBase: return self._cache diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 415284994510e7..91ed050faad104 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -21,8 +21,10 @@ Type, ) +from airbyte_lib import exceptions as exc from airbyte_lib._factories.cache_factories import get_default_cache from airbyte_lib._util import protocol_util # Internal utility functions +from airbyte_lib.datasets._lazy import LazyDataset from airbyte_lib.results import ReadResult from airbyte_lib.telemetry import ( CacheTelemetryInfo, @@ -70,26 +72,35 @@ def __init__( self._processed_records = 0 self.executor = executor self.name = name - self.streams: list[str] | None = None self._processed_records = 0 self._config_dict: dict[str, Any] | None = None self._last_log_messages: list[str] = [] self._discovered_catalog: AirbyteCatalog | None = None self._spec: ConnectorSpecification | None = None + self._selected_stream_names: list[str] | None = None if config is not None: self.set_config(config) if streams is not None: self.set_streams(streams) def set_streams(self, streams: list[str]) -> None: + """Optionally, select the stream names that should be read from the connector. + + Currently, if this is not set, all streams will be read. + + TODO: In the future if not set, the default behavior may exclude streams which the connector + would default to disabled. (For instance, streams that require a premium license + are sometimes disabled by default within the connector.) + """ available_streams = self.get_available_streams() for stream in streams: if stream not in available_streams: - raise Exception( - f"Stream {stream} is not available for connector {self.name}. " - f"Choose from: {available_streams}", + raise exc.AirbyteStreamNotFoundError( + stream_name=stream, + connector_name=self.name, + available_streams=available_streams, ) - self.streams = streams + self._selected_stream_names = streams def set_config(self, config: dict[str, Any]) -> None: self._validate_config(config) @@ -98,8 +109,8 @@ def set_config(self, config: dict[str, Any]) -> None: @property def _config(self) -> dict[str, Any]: if self._config_dict is None: - raise Exception( - "Config is not set, either set in get_connector or via source.set_config", + raise exc.AirbyteConnectorConfigurationMissingError( + guidance="Provide via get_connector() or set_config()" ) return self._config_dict @@ -116,8 +127,8 @@ def _discover(self) -> AirbyteCatalog: for msg in self._execute(["discover", "--config", config_file]): if msg.type == Type.CATALOG and msg.catalog: return msg.catalog - raise Exception( - f"Connector did not return a catalog. Last logs: {self._last_log_messages}", + raise exc.AirbyteConnectorMissingCatalogError( + log_text=self._last_log_messages, ) def _validate_config(self, config: dict[str, Any]) -> None: @@ -127,7 +138,7 @@ def _validate_config(self, config: dict[str, Any]) -> None: def get_available_streams(self) -> list[str]: """Get the available streams from the spec.""" - return [s.name for s in self._discover().streams] + return [s.name for s in self.discovered_catalog.streams] def _get_spec(self, *, force_refresh: bool = False) -> ConnectorSpecification: """Call spec on the connector. @@ -146,35 +157,50 @@ def _get_spec(self, *, force_refresh: bool = False) -> ConnectorSpecification: if self._spec: return self._spec - raise Exception( - f"Connector did not return a spec. Last logs: {self._last_log_messages}", + raise exc.AirbyteConnectorMissingSpecError( + log_text=self._last_log_messages, ) @property - def raw_catalog(self) -> AirbyteCatalog: - """Get the raw catalog for the given streams.""" - return self._discover() + def discovered_catalog(self) -> AirbyteCatalog: + """Get the raw catalog for the given streams. - @property - def configured_catalog(self) -> ConfiguredAirbyteCatalog: - """Get the configured catalog for the given streams.""" + If the catalog is not yet known, we call discover to get it. + """ if self._discovered_catalog is None: self._discovered_catalog = self._discover() + return self._discovered_catalog + + @property + def configured_catalog(self) -> ConfiguredAirbyteCatalog: + """Get the configured catalog for the given streams. + + If the raw catalog is not yet known, we call discover to get it. + + If no specific streams are selected, we return a catalog that syncs all available streams. + + TODO: We should consider disabling by default the streams that the connector would + disable by default. (For instance, streams that require a premium license are sometimes + disabled by default within the connector.) + """ + _ = self.discovered_catalog # Ensure discovered catalog is cached before we start + streams_filter: list[str] | None = self._selected_stream_names return ConfiguredAirbyteCatalog( streams=[ + # TODO: Set sync modes and primary key to a sensible adaptive default ConfiguredAirbyteStream( stream=s, sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, primary_key=None, ) - for s in self._discovered_catalog.streams - if self.streams is None or s.name in self.streams + for s in self.discovered_catalog.streams + if streams_filter is None or s.name in streams_filter ], ) - def get_records(self, stream: str) -> Iterator[dict[str, Any]]: + def get_records(self, stream: str) -> LazyDataset: """Read a stream from the connector. This involves the following steps: @@ -198,15 +224,19 @@ def get_records(self, stream: str) -> Iterator[dict[str, Any]]: ], ) if len(configured_catalog.streams) == 0: - raise ValueError( - f"Stream {stream} is not available for connector {self.name}, " - f"choose from {self.get_available_streams()}", - ) - - iterator: Iterable[dict[str, Any]] = protocol_util.airbyte_messages_to_record_dicts( + raise exc.AirbyteLibInputError( + message="Requested stream does not exist.", + context={ + "stream": stream, + "available_streams": self.get_available_streams(), + "connector_name": self.name, + }, + ) from KeyError(stream) + + iterator: Iterator[dict[str, Any]] = protocol_util.airbyte_messages_to_record_dicts( self._read_with_catalog(streaming_cache_info, configured_catalog), ) - yield from iterator # TODO: Refactor to use LazyDataset here + return LazyDataset(iterator) def check(self) -> None: """Call check on the connector. @@ -223,12 +253,12 @@ def check(self) -> None: if msg.connectionStatus.status != Status.FAILED: return # Success! - raise Exception( - f"Connector returned failed status: {msg.connectionStatus.message}", + raise exc.AirbyteConnectorCheckFailedError( + context={ + "message": msg.connectionStatus.message, + } ) - raise Exception( - f"Connector did not return check status. Last logs: {self._last_log_messages}", - ) + raise exc.AirbyteConnectorCheckFailedError(log_text=self._last_log_messages) def install(self) -> None: """Install the connector if it is not yet installed.""" @@ -253,19 +283,10 @@ def _read(self, cache_info: CacheTelemetryInfo) -> Iterable[AirbyteRecordMessage * execute the connector with read --config --catalog * Listen to the messages and return the AirbyteRecordMessages that come along. """ - catalog = self._discover() - configured_catalog = ConfiguredAirbyteCatalog( - streams=[ - ConfiguredAirbyteStream( - stream=s, - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.overwrite, - ) - for s in catalog.streams - if self.streams is None or s.name in self.streams - ], - ) - yield from self._read_with_catalog(cache_info, configured_catalog) + # Ensure discovered and configured catalog properties are cached before we start reading + _ = self.discovered_catalog + _ = self.configured_catalog + yield from self._read_with_catalog(cache_info, catalog=self.configured_catalog) def _read_with_catalog( self, @@ -330,7 +351,9 @@ def _execute(self, args: list[str]) -> Iterator[AirbyteMessage]: except Exception: self._add_to_logs(line) except Exception as e: - raise Exception(f"Execution failed. Last logs: {self._last_log_messages}") from e + raise exc.AirbyteConnectorReadError( + log_text=self._last_log_messages, + ) from e def _tally_records( self, diff --git a/airbyte-lib/airbyte_lib/validate.py b/airbyte-lib/airbyte_lib/validate.py index 8eac20e1692b43..75eab7e3fd3941 100644 --- a/airbyte-lib/airbyte_lib/validate.py +++ b/airbyte-lib/airbyte_lib/validate.py @@ -3,6 +3,7 @@ This tool checks if connectors are compatible with airbyte-lib. """ +from __future__ import annotations import argparse import json @@ -15,6 +16,7 @@ import yaml import airbyte_lib as ab +from airbyte_lib import exceptions as exc def _parse_args() -> argparse.Namespace: @@ -25,11 +27,16 @@ def _parse_args() -> argparse.Namespace: required=True, help="Path to the connector directory", ) + parser.add_argument( + "--validate-install-only", + action="store_true", + help="Only validate that the connector can be installed and config can be validated.", + ) parser.add_argument( "--sample-config", type=str, - required=True, - help="Path to the sample config.json file", + required=False, + help="Path to the sample config.json file. Required without --validate-install-only.", ) return parser.parse_args() @@ -37,10 +44,13 @@ def _parse_args() -> argparse.Namespace: def _run_subprocess_and_raise_on_failure(args: list[str]) -> None: result = subprocess.run(args, check=False) if result.returncode != 0: - raise Exception(f"{args} exited with code {result.returncode}") + raise exc.AirbyteSubprocessFailedError( + run_args=args, + exit_code=result.returncode, + ) -def tests(connector_name: str, sample_config: str) -> None: +def full_tests(connector_name: str, sample_config: str) -> None: print("Creating source and validating spec and version...") source = ab.get_connector( # TODO: FIXME: noqa: SIM115, PTH123 @@ -61,10 +71,20 @@ def tests(connector_name: str, sample_config: str) -> None: record = next(source.get_records(stream)) assert record, "No record returned" break - except Exception as e: + except exc.AirbyteError as e: print(f"Could not read from stream {stream}: {e}") + except Exception as e: + print(f"Unhandled error occurred when trying to read from {stream}: {e}") else: - raise Exception(f"Could not read from any stream from {streams}") + raise exc.AirbyteNoDataFromConnectorError( + context={"selected_streams": streams}, + ) + + +def install_only_test(connector_name: str) -> None: + print("Creating source and validating spec is returned successfully...") + source = ab.get_connector(connector_name) + source._get_spec(force_refresh=True) # noqa: SLF001 def run() -> None: @@ -82,10 +102,11 @@ def run() -> None: args = _parse_args() connector_dir = args.connector_dir sample_config = args.sample_config - validate(connector_dir, sample_config) + validate_install_only = args.validate_install_only + validate(connector_dir, sample_config, validate_install_only=validate_install_only) -def validate(connector_dir: str, sample_config: str) -> None: +def validate(connector_dir: str, sample_config: str, *, validate_install_only: bool) -> None: # read metadata.yaml metadata_path = Path(connector_dir) / "metadata.yaml" with Path(metadata_path).open() as stream: @@ -118,4 +139,11 @@ def validate(connector_dir: str, sample_config: str) -> None: temp_file.write(json.dumps(registry)) temp_file.seek(0) os.environ["AIRBYTE_LOCAL_REGISTRY"] = str(temp_file.name) - tests(connector_name, sample_config) + if validate_install_only: + install_only_test(connector_name) + else: + if not sample_config: + raise exc.AirbyteLibInputError( + input_value="--sample-config is required without --validate-install-only set" + ) + full_tests(connector_name, sample_config) diff --git a/airbyte-lib/airbyte_lib/version.py b/airbyte-lib/airbyte_lib/version.py index 9ed83a5ef4569f..114a730a5e7c1a 100644 --- a/airbyte-lib/airbyte_lib/version.py +++ b/airbyte-lib/airbyte_lib/version.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import importlib.metadata diff --git a/airbyte-lib/docs.py b/airbyte-lib/docs.py index bfd30c05e554fb..be5dea69b9efe8 100644 --- a/airbyte-lib/docs.py +++ b/airbyte-lib/docs.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import os import pathlib @@ -23,7 +24,7 @@ def run() -> None: # All folders in `airbyte_lib` that don't start with "_" are treated as public modules. for d in os.listdir("airbyte_lib"): dir_path = pathlib.Path(f"airbyte_lib/{d}") - if dir_path.is_dir() and not d.startswith("_"): + if dir_path.is_dir() and not d.startswith("_") and (dir_path / "__init__.py").exists(): public_modules.append(dir_path) pdoc.render.configure(template_directory="docs", show_source=False, search=False) diff --git a/airbyte-lib/docs/generated/airbyte_lib.html b/airbyte-lib/docs/generated/airbyte_lib.html index 240492002ff83c..91c67884b51062 100644 --- a/airbyte-lib/docs/generated/airbyte_lib.html +++ b/airbyte-lib/docs/generated/airbyte_lib.html @@ -1,5 +1,255 @@
+
+
+ + class + CachedDataset(airbyte_lib.datasets._sql.SQLDataset): + + +
+ + +

A dataset backed by a SQL table cache.

+ +

Because this dataset includes all records from the underlying table, we also expose the +underlying table as a SQLAlchemy Table object.

+
+ + +
+
+ + CachedDataset(cache: 'SQLCacheBase', stream_name: str) + + +
+ + + + +
+
+
+
@overrides
+ + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
+ + +

Return a pandas DataFrame representation of the dataset.

+ +

The base implementation simply passes the record iterator to Panda's DataFrame constructor.

+
+ + +
+
+
+ + def + to_sql_table(self) -> 'Table': + + +
+ + + + +
+
+
Inherited Members
+
+
airbyte_lib.datasets._sql.SQLDataset
+
stream_name
+
with_filter
+ +
+
+
+
+
+
+ + class + DuckDBCache(airbyte_lib.caches.duckdb.DuckDBCacheBase): + + +
+ + +

A DuckDB implementation of the cache.

+ +

Parquet is used for local file storage before bulk loading. +Unlike the Snowflake implementation, we can't use the COPY command to load data +so we insert as values instead.

+
+ + +
+
+ file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
+ + + + +
+
+
Inherited Members
+
+
airbyte_lib.caches.base.SQLCacheBase
+
SQLCacheBase
+
type_converter_class
+
use_singleton_connection
+
config
+
file_writer
+
type_converter
+
get_sql_alchemy_url
+
database_name
+
get_sql_engine
+
get_sql_connection
+
get_sql_table_name
+
get_sql_table
+
streams
+
get_records
+
get_pandas_dataframe
+ +
+
airbyte_lib.caches.duckdb.DuckDBCacheBase
+
config_class
+
supports_merge_insert
+
get_telemetry_info
+ +
+
airbyte_lib._processors.RecordProcessor
+
skip_finalize_step
+
source_catalog
+
register_source
+
process_stdin
+
process_input_stream
+
process_airbyte_messages
+ +
+
+
+
+
+
+ + class + DuckDBCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
+ + +

Configuration for the DuckDB cache.

+ +

Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

+
+ + +
+
+ db_path: pathlib.Path | str + + +
+ + +

Normally db_path is a Path object.

+ +

There are some cases, such as when connecting to MotherDuck, where it could be a string that +is not also a path, such as "md:" to connect the user's default MotherDuck DB.

+
+ + +
+
+
+ schema_name: str + + +
+ + +

The name of the schema to write to. Defaults to "main".

+
+ + +
+
+
+
@overrides
+ + def + get_sql_alchemy_url(self) -> str: + + +
+ + +

Return the SQLAlchemy URL to use.

+
+ + +
+
+
+ + def + get_database_name(self) -> str: + + +
+ + +

Return the name of the database.

+
+ + +
+
+
Inherited Members
+
+
pydantic.main.BaseModel
+
BaseModel
+
Config
+
dict
+
json
+
parse_obj
+
parse_raw
+
parse_file
+
from_orm
+
construct
+
copy
+
schema
+
schema_json
+
validate
+
update_forward_refs
+ +
+
airbyte_lib.caches.base.SQLCacheConfigBase
+
dedupe_mode
+
table_prefix
+
table_suffix
+ +
+
airbyte_lib._file_writers.base.FileWriterConfigBase
+
cache_dir
+
cleanup
+ +
+
+
+
@@ -34,7 +284,7 @@
def - get_default_cache() -> airbyte_lib.caches.duckdb.DuckDBCache: + get_default_cache() -> DuckDBCache:
@@ -52,7 +302,7 @@
def - new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> airbyte_lib.caches.duckdb.DuckDBCache: + new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> DuckDBCache:
@@ -70,61 +320,6 @@
-
-
-
- - class - CachedDataset(abc.ABC, collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]): - - -
- - -

Base implementation for all datasets.

-
- - -
-
- - CachedDataset(cache: airbyte_lib.caches.base.SQLCacheBase, stream: str) - - -
- - - - -
-
-
- - def - to_pandas(self) -> pandas.core.frame.DataFrame: - - -
- - -

Return a pandas DataFrame representation of the dataset.

-
- - -
-
-
- - def - to_sql_table(self) -> sqlalchemy.sql.schema.Table: - - -
- - - - -
@@ -173,6 +368,17 @@ +
+
+
+ streams: collections.abc.Mapping[str, CachedDataset] + + +
+ + + +
@@ -233,17 +439,6 @@ -
-
-
- streams: list[str] | None - - -
- - - -
@@ -255,7 +450,15 @@
- +

Optionally, select the stream names that should be read from the connector.

+ +

Currently, if this is not set, all streams will be read.

+ +

TODO: In the future if not set, the default behavior may exclude streams which the connector +would default to disabled. (For instance, streams that require a premium license +are sometimes disabled by default within the connector.)

+
+
@@ -286,15 +489,17 @@
-
+
- raw_catalog: airbyte_protocol.models.airbyte_protocol.AirbyteCatalog + discovered_catalog: airbyte_protocol.models.airbyte_protocol.AirbyteCatalog
- +

Get the raw catalog for the given streams.

+ +

If the catalog is not yet known, we call discover to get it.

@@ -308,6 +513,14 @@

Get the configured catalog for the given streams.

+ +

If the raw catalog is not yet known, we call discover to get it.

+ +

If no specific streams are selected, we return a catalog that syncs all available streams.

+ +

TODO: We should consider disabling by default the streams that the connector would +disable by default. (For instance, streams that require a premium license are sometimes +disabled by default within the connector.)

@@ -316,7 +529,7 @@
def - get_records(self, stream: str) -> collections.abc.Iterator[dict[str, typing.Any]]: + get_records(self, stream: str) -> airbyte_lib.datasets._lazy.LazyDataset:
diff --git a/airbyte-lib/docs/generated/airbyte_lib/caches.html b/airbyte-lib/docs/generated/airbyte_lib/caches.html index dbb675f6d5a204..60dd860c4aa8bd 100644 --- a/airbyte-lib/docs/generated/airbyte_lib/caches.html +++ b/airbyte-lib/docs/generated/airbyte_lib/caches.html @@ -622,7 +622,7 @@
Inherited Members
- streams: dict[str, airbyte_lib.datasets._base.DatasetBase] + streams: dict[str, airbyte_lib.datasets._sql.CachedDataset]
@@ -637,15 +637,13 @@
Inherited Members
def - get_records( self, stream_name: str) -> collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]: + get_records(self, stream_name: str) -> airbyte_lib.datasets._sql.CachedDataset:

Uses SQLAlchemy to select all rows from the table.

- -

TODO: Refactor to return a LazyDataset here.

diff --git a/airbyte-lib/docs/generated/airbyte_lib/datasets.html b/airbyte-lib/docs/generated/airbyte_lib/datasets.html index acd3bb5928b63e..76089344eca0b1 100644 --- a/airbyte-lib/docs/generated/airbyte_lib/datasets.html +++ b/airbyte-lib/docs/generated/airbyte_lib/datasets.html @@ -4,20 +4,23 @@
class - CachedDataset(abc.ABC, collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]): + CachedDataset(airbyte_lib.datasets.SQLDataset):
-

Base implementation for all datasets.

+

A dataset backed by a SQL table cache.

+ +

Because this dataset includes all records from the underlying table, we also expose the +underlying table as a SQLAlchemy Table object.

- CachedDataset(cache: airbyte_lib.caches.base.SQLCacheBase, stream: str) + CachedDataset(cache: 'SQLCacheBase', stream_name: str)
@@ -28,7 +31,8 @@
- +
@overrides
+ def to_pandas(self) -> pandas.core.frame.DataFrame: @@ -37,6 +41,8 @@

Return a pandas DataFrame representation of the dataset.

+ +

The base implementation simply passes the record iterator to Panda's DataFrame constructor.

@@ -45,7 +51,7 @@
def - to_sql_table(self) -> sqlalchemy.sql.schema.Table: + to_sql_table(self) -> 'Table':
@@ -53,13 +59,23 @@ +
+
+
Inherited Members
+
+ +
class - DatasetBase(abc.ABC, collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]): + DatasetBase(abc.ABC):
@@ -80,6 +96,8 @@

Return a pandas DataFrame representation of the dataset.

+ +

The base implementation simply passes the record iterator to Panda's DataFrame constructor.

@@ -112,6 +130,127 @@
Inherited Members
+
+
+ + class + LazyDataset(airbyte_lib.datasets.DatasetBase): + + +
+ + +

A dataset that is loaded incrementally from a source or a SQL query.

+
+ + +
+
+ + LazyDataset( iterator: collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]) + + +
+ + + + +
+
+
Inherited Members
+
+ +
+
+
+
+
+ + class + SQLDataset(airbyte_lib.datasets.DatasetBase): + + +
+ + +

A dataset that is loaded incrementally from a SQL query.

+ +

The CachedDataset class is a subclass of this class, which simply passes a SELECT over the full +table as the query statement.

+
+ + +
+
+ + SQLDataset( cache: 'SQLCacheBase', stream_name: str, query_statement: 'Selectable') + + +
+ + + + +
+
+
+ stream_name: str + + +
+ + + + +
+
+
+ + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
+ + +

Return a pandas DataFrame representation of the dataset.

+ +

The base implementation simply passes the record iterator to Panda's DataFrame constructor.

+
+ + +
+
+
+ + def + with_filter( self, *filter_expressions: 'ClauseElement | str') -> SQLDataset: + + +
+ + +

Filter the dataset by a set of column values.

+ +

Filters can be specified as either a string or a SQLAlchemy expression.

+ +

Filters are lazily applied to the dataset, so they can be chained together. For example:

+ +
    dataset.with_filter("id > 5").with_filter("id < 10")
+
+ +

is equivalent to:

+ +
    dataset.with_filter("id > 5", "id < 10")
+
+
+ + +
+
diff --git a/airbyte-lib/examples/run_snowflake_faker.py b/airbyte-lib/examples/run_snowflake_faker.py index 5a6084ce16dcf5..2102ab658b179d 100644 --- a/airbyte-lib/examples/run_snowflake_faker.py +++ b/airbyte-lib/examples/run_snowflake_faker.py @@ -1,5 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. - +from __future__ import annotations import json import os diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py index 46c8dee411d878..e1952157738198 100644 --- a/airbyte-lib/examples/run_spacex.py +++ b/airbyte-lib/examples/run_spacex.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations from itertools import islice diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py index de57ca8420ff6b..a984bd6d652f74 100644 --- a/airbyte-lib/examples/run_test_source.py +++ b/airbyte-lib/examples/run_test_source.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import os diff --git a/airbyte-lib/examples/run_test_source_single_stream.py b/airbyte-lib/examples/run_test_source_single_stream.py index b1cc55cd1f5c70..88faec90cb5fe5 100644 --- a/airbyte-lib/examples/run_test_source_single_stream.py +++ b/airbyte-lib/examples/run_test_source_single_stream.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import os diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock index 06576ed330a2f0..5dafb19e798a12 100644 --- a/airbyte-lib/poetry.lock +++ b/airbyte-lib/poetry.lock @@ -567,13 +567,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.26.2" +version = "2.27.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, - {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, ] [package.dependencies] @@ -879,77 +879,73 @@ format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-va [[package]] name = "markupsafe" -version = "2.1.3" +version = "2.1.4" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" files = [ - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, - {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, -] - -[[package]] -name = "mirakuru" -version = "2.5.2" -description = "Process executor (not only) for tests." -optional = false -python-versions = ">=3.8" -files = [ - {file = "mirakuru-2.5.2-py3-none-any.whl", hash = "sha256:90c2d90a8cf14349b2f33e6db30a16acd855499811e0312e56cf80ceacf2d3e5"}, - {file = "mirakuru-2.5.2.tar.gz", hash = "sha256:41ca583d355eb7a6cfdc21c1aea549979d685c27b57239b88725434f115a7132"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de8153a7aae3835484ac168a9a9bdaa0c5eee4e0bc595503c95d53b942879c84"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e888ff76ceb39601c59e219f281466c6d7e66bd375b4ec1ce83bcdc68306796b"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0b838c37ba596fcbfca71651a104a611543077156cb0a26fe0c475e1f152ee8"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac1ebf6983148b45b5fa48593950f90ed6d1d26300604f321c74a9ca1609f8e"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbad3d346df8f9d72622ac71b69565e621ada2ce6572f37c2eae8dacd60385d"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5291d98cd3ad9a562883468c690a2a238c4a6388ab3bd155b0c75dd55ece858"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a7cc49ef48a3c7a0005a949f3c04f8baa5409d3f663a1b36f0eba9bfe2a0396e"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b83041cda633871572f0d3c41dddd5582ad7d22f65a72eacd8d3d6d00291df26"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-win32.whl", hash = "sha256:0c26f67b3fe27302d3a412b85ef696792c4a2386293c53ba683a89562f9399b0"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:a76055d5cb1c23485d7ddae533229039b850db711c554a12ea64a0fd8a0129e2"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9e9e3c4020aa2dc62d5dd6743a69e399ce3de58320522948af6140ac959ab863"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0042d6a9880b38e1dd9ff83146cc3c9c18a059b9360ceae207805567aacccc69"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d03fea4c4e9fd0ad75dc2e7e2b6757b80c152c032ea1d1de487461d8140efc"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ab3a886a237f6e9c9f4f7d272067e712cdb4efa774bef494dccad08f39d8ae6"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abf5ebbec056817057bfafc0445916bb688a255a5146f900445d081db08cbabb"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e1a0d1924a5013d4f294087e00024ad25668234569289650929ab871231668e7"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e7902211afd0af05fbadcc9a312e4cf10f27b779cf1323e78d52377ae4b72bea"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c669391319973e49a7c6230c218a1e3044710bc1ce4c8e6eb71f7e6d43a2c131"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-win32.whl", hash = "sha256:31f57d64c336b8ccb1966d156932f3daa4fee74176b0fdc48ef580be774aae74"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:54a7e1380dfece8847c71bf7e33da5d084e9b889c75eca19100ef98027bd9f56"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a76cd37d229fc385738bd1ce4cba2a121cf26b53864c1772694ad0ad348e509e"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:987d13fe1d23e12a66ca2073b8d2e2a75cec2ecb8eab43ff5624ba0ad42764bc"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5244324676254697fe5c181fc762284e2c5fceeb1c4e3e7f6aca2b6f107e60dc"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78bc995e004681246e85e28e068111a4c3f35f34e6c62da1471e844ee1446250"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4d176cfdfde84f732c4a53109b293d05883e952bbba68b857ae446fa3119b4f"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f9917691f410a2e0897d1ef99619fd3f7dd503647c8ff2475bf90c3cf222ad74"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f06e5a9e99b7df44640767842f414ed5d7bedaaa78cd817ce04bbd6fd86e2dd6"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:396549cea79e8ca4ba65525470d534e8a41070e6b3500ce2414921099cb73e8d"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-win32.whl", hash = "sha256:f6be2d708a9d0e9b0054856f07ac7070fbe1754be40ca8525d5adccdbda8f475"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:5045e892cfdaecc5b4c01822f353cf2c8feb88a6ec1c0adef2a2e705eef0f656"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a07f40ef8f0fbc5ef1000d0c78771f4d5ca03b4953fc162749772916b298fc4"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d18b66fe626ac412d96c2ab536306c736c66cf2a31c243a45025156cc190dc8a"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:698e84142f3f884114ea8cf83e7a67ca8f4ace8454e78fe960646c6c91c63bfa"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a3b78a5af63ec10d8604180380c13dcd870aba7928c1fe04e881d5c792dc4e"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:15866d7f2dc60cfdde12ebb4e75e41be862348b4728300c36cdf405e258415ec"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6aa5e2e7fc9bc042ae82d8b79d795b9a62bd8f15ba1e7594e3db243f158b5565"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54635102ba3cf5da26eb6f96c4b8c53af8a9c0d97b64bdcb592596a6255d8518"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-win32.whl", hash = "sha256:3583a3a3ab7958e354dc1d25be74aee6228938312ee875a22330c4dc2e41beb0"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-win_amd64.whl", hash = "sha256:d6e427c7378c7f1b2bef6a344c925b8b63623d3321c09a237b7cc0e77dd98ceb"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bf1196dcc239e608605b716e7b166eb5faf4bc192f8a44b81e85251e62584bd2"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df98d4a9cd6a88d6a585852f56f2155c9cdb6aec78361a19f938810aa020954"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b835aba863195269ea358cecc21b400276747cc977492319fd7682b8cd2c253d"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23984d1bdae01bee794267424af55eef4dfc038dc5d1272860669b2aa025c9e3"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c98c33ffe20e9a489145d97070a435ea0679fddaabcafe19982fe9c971987d5"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9896fca4a8eb246defc8b2a7ac77ef7553b638e04fbf170bff78a40fa8a91474"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b0fe73bac2fed83839dbdbe6da84ae2a31c11cfc1c777a40dbd8ac8a6ed1560f"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c7556bafeaa0a50e2fe7dc86e0382dea349ebcad8f010d5a7dc6ba568eaaa789"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-win32.whl", hash = "sha256:fc1a75aa8f11b87910ffd98de62b29d6520b6d6e8a3de69a70ca34dea85d2a8a"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-win_amd64.whl", hash = "sha256:3a66c36a3864df95e4f62f9167c734b3b1192cb0851b43d7cc08040c074c6279"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:765f036a3d00395a326df2835d8f86b637dbaf9832f90f5d196c3b8a7a5080cb"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:21e7af8091007bf4bebf4521184f4880a6acab8df0df52ef9e513d8e5db23411"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5c31fe855c77cad679b302aabc42d724ed87c043b1432d457f4976add1c2c3e"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7653fa39578957bc42e5ebc15cf4361d9e0ee4b702d7d5ec96cdac860953c5b4"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47bb5f0142b8b64ed1399b6b60f700a580335c8e1c57f2f15587bd072012decc"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fe8512ed897d5daf089e5bd010c3dc03bb1bdae00b35588c49b98268d4a01e00"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:36d7626a8cca4d34216875aee5a1d3d654bb3dac201c1c003d182283e3205949"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b6f14a9cd50c3cb100eb94b3273131c80d102e19bb20253ac7bd7336118a673a"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-win32.whl", hash = "sha256:c8f253a84dbd2c63c19590fa86a032ef3d8cc18923b8049d91bcdeeb2581fbf6"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:8b570a1537367b52396e53325769608f2a687ec9a4363647af1cded8928af959"}, + {file = "MarkupSafe-2.1.4.tar.gz", hash = "sha256:3aae9af4cac263007fd6309c64c6ab4506dd2b79382d9d19a1994f9240b8db4f"}, ] -[package.dependencies] -psutil = {version = ">=4.0.0", markers = "sys_platform != \"cygwin\""} - [[package]] name = "mypy" version = "1.8.0" @@ -1114,13 +1110,13 @@ files = [ [[package]] name = "overrides" -version = "7.4.0" +version = "7.6.0" description = "A decorator to automatically detect mismatch when overriding a method." optional = false python-versions = ">=3.6" files = [ - {file = "overrides-7.4.0-py3-none-any.whl", hash = "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d"}, - {file = "overrides-7.4.0.tar.gz", hash = "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"}, + {file = "overrides-7.6.0-py3-none-any.whl", hash = "sha256:c36e6635519ea9c5b043b65c36d4b886aee8bd45b7d4681d2a6df0898df4b654"}, + {file = "overrides-7.6.0.tar.gz", hash = "sha256:01e15bbbf15b766f0675c275baa1878bd1c7dc9bc7b9ee13e677cdba93dc1bd9"}, ] [[package]] @@ -1287,30 +1283,19 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co [[package]] name = "pluggy" -version = "1.3.0" +version = "1.4.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, - {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, ] [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] -[[package]] -name = "port-for" -version = "0.7.2" -description = "Utility that helps with local TCP ports management. It can find an unused TCP localhost port and remember the association." -optional = false -python-versions = ">=3.8" -files = [ - {file = "port-for-0.7.2.tar.gz", hash = "sha256:074f29335130578aa42fef3726985e57d01c15189e509633a8a1b0b7f9226349"}, - {file = "port_for-0.7.2-py3-none-any.whl", hash = "sha256:16b279ab4f210bad33515c45bd9af0c6e048ab24c3b6bbd9cfc7e451782617df"}, -] - [[package]] name = "proto-plus" version = "1.23.0" @@ -1349,166 +1334,84 @@ files = [ ] [[package]] -name = "psutil" -version = "5.9.7" -description = "Cross-platform lib for process and system monitoring in Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6"}, - {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056"}, - {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508"}, - {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df"}, - {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7"}, - {file = "psutil-5.9.7-cp27-none-win32.whl", hash = "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c"}, - {file = "psutil-5.9.7-cp27-none-win_amd64.whl", hash = "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6"}, - {file = "psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e"}, - {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284"}, - {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"}, - {file = "psutil-5.9.7-cp36-cp36m-win32.whl", hash = "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9"}, - {file = "psutil-5.9.7-cp36-cp36m-win_amd64.whl", hash = "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e"}, - {file = "psutil-5.9.7-cp37-abi3-win32.whl", hash = "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68"}, - {file = "psutil-5.9.7-cp37-abi3-win_amd64.whl", hash = "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414"}, - {file = "psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340"}, - {file = "psutil-5.9.7.tar.gz", hash = "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c"}, -] - -[package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] - -[[package]] -name = "psycopg" -version = "3.1.17" -description = "PostgreSQL database adapter for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "psycopg-3.1.17-py3-none-any.whl", hash = "sha256:96b7b13af6d5a514118b759a66b2799a8a4aa78675fa6bb0d3f7d52d67eff002"}, - {file = "psycopg-3.1.17.tar.gz", hash = "sha256:437e7d7925459f21de570383e2e10542aceb3b9cb972ce957fdd3826ca47edc6"}, -] - -[package.dependencies] -psycopg-binary = {version = "3.1.17", optional = true, markers = "implementation_name != \"pypy\" and extra == \"binary\""} -psycopg-pool = {version = "*", optional = true, markers = "extra == \"pool\""} -typing-extensions = ">=4.1" -tzdata = {version = "*", markers = "sys_platform == \"win32\""} - -[package.extras] -binary = ["psycopg-binary (==3.1.17)"] -c = ["psycopg-c (==3.1.17)"] -dev = ["black (>=23.1.0)", "codespell (>=2.2)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.4.1)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] -docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] -pool = ["psycopg-pool"] -test = ["anyio (>=3.6.2,<4.0)", "mypy (>=1.4.1)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] - -[[package]] -name = "psycopg-binary" -version = "3.1.17" -description = "PostgreSQL database adapter for Python -- C optimisation distribution" -optional = false -python-versions = ">=3.7" -files = [ - {file = "psycopg_binary-3.1.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f9ba559eabb0ba1afd4e0504fa0b10e00a212cac0c4028b8a1c3b087b5c1e5de"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2b2a689eaede08cf91a36b10b0da6568dd6e4669200f201e082639816737992b"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a16abab0c1abc58feb6ab11d78d0f8178a67c3586bd70628ec7c0218ec04c4ef"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73e7097b81cad9ae358334e3cec625246bb3b8013ae6bb287758dd6435e12f65"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:67a5b93101bc85a95a189c0a23d02a29cf06c1080a695a0dedfdd50dd734662a"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:751b31c2faae0348f87f22b45ef58f704bdcfc2abdd680fa0c743c124071157e"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b447ea765e71bc33a82cf070bba814b1efa77967442d116b95ccef8ce5da7631"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d2e9ed88d9a6a475c67bf70fc8285e88ccece0391727c7701e5a512e0eafbb05"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a89f36bf7b612ff6ed3e789bd987cbd0787cf0d66c49386fa3bad816dd7bee87"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ccbe8b2ec444763a51ecb1213befcbb75defc1ef36e7dd5dff501a23d7ce8cf"}, - {file = "psycopg_binary-3.1.17-cp310-cp310-win_amd64.whl", hash = "sha256:adb670031b27949c9dc5cf585c4a5a6b4469d3879fd2fb9d39b6d53e5f66b9bc"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0227885686c2cc0104ceb22d6eebc732766e9ad48710408cb0123237432e5435"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9124b6db07e8d8b11f4512b8b56cbe136bf1b7d0417d1280e62291a9dcad4408"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a46f77ba0ca7c5a5449b777170a518fa7820e1710edb40e777c9798f00d033"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f5f5bcbb772d8c243d605fc7151beec760dd27532d42145a58fb74ef9c5fbf2"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:267a82548c21476120e43dc72b961f1af52c380c0b4c951bdb34cf14cb26bd35"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b20013051f1fd7d02b8d0766cfe8d009e8078babc00a6d39bc7e2d50a7b96af"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c5c38129cc79d7e3ba553035b9962a442171e9f97bb1b8795c0885213f206f3"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d01c4faae66de60fcd3afd3720dcc8ffa03bc2087f898106da127774db12aac5"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e6ae27b0617ad3809449964b5e901b21acff8e306abacb8ba71d5ee7c8c47eeb"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:40af298b209dd77ca2f3e7eb3fbcfb87a25999fc015fcd14140bde030a164c7e"}, - {file = "psycopg_binary-3.1.17-cp311-cp311-win_amd64.whl", hash = "sha256:7b4e4c2b05f3b431e9026e82590b217e87696e7a7548f512ae8059d59fa8af3b"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ea425a8dcd808a7232a5417d2633bfa543da583a2701b5228e9e29989a50deda"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3f1196d76860e72d338fab0d2b6722e8d47e2285d693e366ae36011c4a5898a"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1e867c2a729348df218a14ba1b862e627177fd57c7b4f3db0b4c708f6d03696"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0711e46361ea3047cd049868419d030c8236a9dea7e9ed1f053cbd61a853ec9"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1c0115bdf80cf6c8c9109cb10cf6f650fd1a8d841f884925e8cb12f34eb5371"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d0d154c780cc7b28a3a0886e8a4b18689202a1dbb522b3c771eb3a1289cf7c3"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f4028443bf25c1e04ecffdc552c0a98d826903dec76a1568dfddf5ebbbb03db7"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf424d92dd7e94705b31625b02d396297a7c8fab4b6f7de8dba6388323a7b71c"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:00377f6963ee7e4bf71cab17c2c235ef0624df9483f3b615d86aa24cde889d42"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9690a535d9ccd361bbc3590bfce7fe679e847f44fa7cc97f3b885f4744ca8a2c"}, - {file = "psycopg_binary-3.1.17-cp312-cp312-win_amd64.whl", hash = "sha256:6b2ae342d69684555bfe77aed5546d125b4a99012e0b83a8b3da68c8829f0935"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:86bb3656c8d744cc1e42003414cd6c765117d70aa23da6c0f4ff2b826e0fd0fd"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c10b7713e3ed31df7319c2a72d5fea5a2536476d7695a3e1d18a1f289060997c"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12eab8bc91b4ba01b2ecee3b5b80501934b198f6e1f8d4b13596f3f38ba6e762"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a728beefd89b430ebe2729d04ba10e05036b5e9d01648da60436000d2fcd242"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61104b8e7a43babf2bbaa36c08e31a12023e2f967166e99d6b052b11a4c7db06"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:02cd2eb62ffc56f8c847d68765cbf461b3d11b438fe48951e44b6c563ec27d18"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ca1757a6e080086f7234dc45684e81a47a66a6dd492a37d6ce38c58a1a93e9ff"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:6e3543edc18553e31a3884af3cd7eea43d6c44532d8b9b16f3e743cdf6cfe6c5"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:914254849486e14aa931b0b3382cd16887f1507068ffba775cbdc5a55fe9ef19"}, - {file = "psycopg_binary-3.1.17-cp37-cp37m-win_amd64.whl", hash = "sha256:92fad8f1aa80a5ab316c0493dc6d1b54c1dba21937e43eea7296ff4a0ccc071e"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6d4f2e15d33ed4f9776fdf23683512d76f4e7825c4b80677e9e3ce6c1b193ff2"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4fa26836ce074a1104249378727e1f239a01530f36bae16e77cf6c50968599b4"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d54bcf2dfc0880bf13f38512d44b194c092794e4ee9e01d804bc6cd3eed9bfb7"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e28024204dc0c61094268c682041d2becfedfea2e3b46bed5f6138239304d98"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b1ec6895cab887b92c303565617f994c9b9db53befda81fa2a31b76fe8a3ab1"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:420c1eb1626539c261cf3fbe099998da73eb990f9ce1a34da7feda414012ea5f"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:83404a353240fdff5cfe9080665fdfdcaa2d4d0c5112e15b0a2fe2e59200ed57"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a0c4ba73f9e7721dd6cc3e6953016652dbac206f654229b7a1a8ac182b16e689"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f6898bf1ca5aa01115807643138e3e20ec603b17a811026bc4a49d43055720a7"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6b40fa54a02825d3d6a8009d9a82a2b4fad80387acf2b8fd6d398fd2813cb2d9"}, - {file = "psycopg_binary-3.1.17-cp38-cp38-win_amd64.whl", hash = "sha256:78ebb43dca7d5b41eee543cd005ee5a0256cecc74d84acf0fab4f025997b837e"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:02ac573f5a6e79bb6df512b3a6279f01f033bbd45c47186e8872fee45f6681d0"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:704f6393d758b12a4369887fe956b2a8c99e4aced839d9084de8e3f056015d40"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0340ef87a888fd940796c909e038426f4901046f61856598582a817162c64984"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a880e4113af3ab84d6a0991e3f85a2424924c8a182733ab8d964421df8b5190a"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93921178b9a40c60c26e47eb44970f88c49fe484aaa3bb7ec02bb8b514eab3d9"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a05400e9314fc30bc1364865ba9f6eaa2def42b5e7e67f71f9a4430f870023e"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3e2cc2bbf37ff1cf11e8b871c294e3532636a3cf7f0c82518b7537158923d77b"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a343261701a8f63f0d8268f7fd32be40ffe28d24b65d905404ca03e7281f7bb5"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:dceb3930ec426623c0cacc78e447a90882981e8c49d6fea8d1e48850e24a0170"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d613a23f8928f30acb2b6b2398cb7775ba9852e8968e15df13807ba0d3ebd565"}, - {file = "psycopg_binary-3.1.17-cp39-cp39-win_amd64.whl", hash = "sha256:d90c0531e9d591bde8cea04e75107fcddcc56811b638a34853436b23c9a3cb7d"}, -] - -[[package]] -name = "psycopg-pool" -version = "3.2.1" -description = "Connection Pool for Psycopg" -optional = false -python-versions = ">=3.8" -files = [ - {file = "psycopg-pool-3.2.1.tar.gz", hash = "sha256:6509a75c073590952915eddbba7ce8b8332a440a31e77bba69561483492829ad"}, - {file = "psycopg_pool-3.2.1-py3-none-any.whl", hash = "sha256:060b551d1b97a8d358c668be58b637780b884de14d861f4f5ecc48b7563aafb7"}, -] - -[package.dependencies] -typing-extensions = ">=4.4" - -[[package]] -name = "psycopg2" +name = "psycopg2-binary" version = "2.9.9" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.7" files = [ - {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, - {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, - {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, - {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, - {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, - {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, - {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, - {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, - {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, ] [[package]] @@ -1607,47 +1510,47 @@ files = [ [[package]] name = "pydantic" -version = "1.10.13" +version = "1.10.14" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, - {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, - {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, - {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, - {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, - {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, - {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, - {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, - {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, - {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, ] [package.dependencies] @@ -1827,24 +1730,6 @@ pytest = [ {version = ">=4.6", markers = "python_version >= \"3.6\" and python_version < \"3.10\""}, ] -[[package]] -name = "pytest-postgresql" -version = "5.0.0" -description = "Postgresql fixtures and fixture factories for Pytest." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pytest-postgresql-5.0.0.tar.gz", hash = "sha256:22edcbafab8995ee85b8d948ddfaad4f70c2c7462303d7477ecd2f77fc9d15bd"}, - {file = "pytest_postgresql-5.0.0-py3-none-any.whl", hash = "sha256:6e8f0773b57c9b8975b6392c241b7b81b7018f32079a533f368f2fbda732ecd3"}, -] - -[package.dependencies] -mirakuru = "*" -port-for = ">=0.6.0" -psycopg = ">=3.0.0" -pytest = ">=6.2" -setuptools = "*" - [[package]] name = "python-dateutil" version = "2.8.2" @@ -2154,28 +2039,28 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.1.13" +version = "0.1.14" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:e3fd36e0d48aeac672aa850045e784673449ce619afc12823ea7868fcc41d8ba"}, - {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9fb6b3b86450d4ec6a6732f9f60c4406061b6851c4b29f944f8c9d91c3611c7a"}, - {file = "ruff-0.1.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b13ba5d7156daaf3fd08b6b993360a96060500aca7e307d95ecbc5bb47a69296"}, - {file = "ruff-0.1.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9ebb40442f7b531e136d334ef0851412410061e65d61ca8ce90d894a094feb22"}, - {file = "ruff-0.1.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:226b517f42d59a543d6383cfe03cccf0091e3e0ed1b856c6824be03d2a75d3b6"}, - {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5f0312ba1061e9b8c724e9a702d3c8621e3c6e6c2c9bd862550ab2951ac75c16"}, - {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2f59bcf5217c661254bd6bc42d65a6fd1a8b80c48763cb5c2293295babd945dd"}, - {file = "ruff-0.1.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6894b00495e00c27b6ba61af1fc666f17de6140345e5ef27dd6e08fb987259d"}, - {file = "ruff-0.1.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1600942485c6e66119da294c6294856b5c86fd6df591ce293e4a4cc8e72989"}, - {file = "ruff-0.1.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ee3febce7863e231a467f90e681d3d89210b900d49ce88723ce052c8761be8c7"}, - {file = "ruff-0.1.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dcaab50e278ff497ee4d1fe69b29ca0a9a47cd954bb17963628fa417933c6eb1"}, - {file = "ruff-0.1.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f57de973de4edef3ad3044d6a50c02ad9fc2dff0d88587f25f1a48e3f72edf5e"}, - {file = "ruff-0.1.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7a36fa90eb12208272a858475ec43ac811ac37e91ef868759770b71bdabe27b6"}, - {file = "ruff-0.1.13-py3-none-win32.whl", hash = "sha256:a623349a505ff768dad6bd57087e2461be8db58305ebd5577bd0e98631f9ae69"}, - {file = "ruff-0.1.13-py3-none-win_amd64.whl", hash = "sha256:f988746e3c3982bea7f824c8fa318ce7f538c4dfefec99cd09c8770bd33e6539"}, - {file = "ruff-0.1.13-py3-none-win_arm64.whl", hash = "sha256:6bbbc3042075871ec17f28864808540a26f0f79a4478c357d3e3d2284e832998"}, - {file = "ruff-0.1.13.tar.gz", hash = "sha256:e261f1baed6291f434ffb1d5c6bd8051d1c2a26958072d38dfbec39b3dda7352"}, + {file = "ruff-0.1.14-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:96f76536df9b26622755c12ed8680f159817be2f725c17ed9305b472a757cdbb"}, + {file = "ruff-0.1.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ab3f71f64498c7241123bb5a768544cf42821d2a537f894b22457a543d3ca7a9"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7060156ecc572b8f984fd20fd8b0fcb692dd5d837b7606e968334ab7ff0090ab"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a53d8e35313d7b67eb3db15a66c08434809107659226a90dcd7acb2afa55faea"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bea9be712b8f5b4ebed40e1949379cfb2a7d907f42921cf9ab3aae07e6fba9eb"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2270504d629a0b064247983cbc495bed277f372fb9eaba41e5cf51f7ba705a6a"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80258bb3b8909b1700610dfabef7876423eed1bc930fe177c71c414921898efa"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:653230dd00aaf449eb5ff25d10a6e03bc3006813e2cb99799e568f55482e5cae"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b3acc6c4e6928459ba9eb7459dd4f0c4bf266a053c863d72a44c33246bfdbf"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6b3dadc9522d0eccc060699a9816e8127b27addbb4697fc0c08611e4e6aeb8b5"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1c8eca1a47b4150dc0fbec7fe68fc91c695aed798532a18dbb1424e61e9b721f"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:62ce2ae46303ee896fc6811f63d6dabf8d9c389da0f3e3f2bce8bc7f15ef5488"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b2027dde79d217b211d725fc833e8965dc90a16d0d3213f1298f97465956661b"}, + {file = "ruff-0.1.14-py3-none-win32.whl", hash = "sha256:722bafc299145575a63bbd6b5069cb643eaa62546a5b6398f82b3e4403329cab"}, + {file = "ruff-0.1.14-py3-none-win_amd64.whl", hash = "sha256:e3d241aa61f92b0805a7082bd89a9990826448e4d0398f0e2bc8f05c75c63d99"}, + {file = "ruff-0.1.14-py3-none-win_arm64.whl", hash = "sha256:269302b31ade4cde6cf6f9dd58ea593773a37ed3f7b97e793c8594b262466b67"}, + {file = "ruff-0.1.14.tar.gz", hash = "sha256:ad3f8088b2dfd884820289a06ab718cde7d38b94972212cc4ba90d5fbc9955f3"}, ] [[package]] @@ -2609,4 +2494,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "5457f59d89d371fbdacd1a5ba132de627ad4f7b6bd2c073fe75b54d90d216e5f" +content-hash = "5eba75179b62f56be141db82121ca9e1c623944306172d7bdacaf5388e6f3384" diff --git a/airbyte-lib/poetry.toml b/airbyte-lib/poetry.toml new file mode 100644 index 00000000000000..ab1033bd37224e --- /dev/null +++ b/airbyte-lib/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml index a1aea3d19bd4bb..a4749bbd750bb9 100644 --- a/airbyte-lib/pyproject.toml +++ b/airbyte-lib/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "airbyte-lib" description = "AirbyteLib" -version = "0.1.0" +version = "0.1.0dev.2" authors = ["Airbyte "] readme = "README.md" packages = [{include = "airbyte_lib"}] @@ -14,8 +14,8 @@ airbyte-cdk = "^0.58.3" jsonschema = "3.2.0" orjson = "^3.9.10" overrides = "^7.4.0" -pandas = "^2.1.4" -psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} +pandas = "2.1.4" # 2.2.0 breaks sqlalchemy interop - TODO: optionally retest higher versions +psycopg2-binary = "^2.9.9" python-ulid = "^2.2.0" types-pyyaml = "^6.0.12.12" ulid = "^1.1" @@ -25,7 +25,10 @@ snowflake-sqlalchemy = "^1.5.1" duckdb-engine = "^0.10.0" requests = "^2.31.0" pyarrow = "^14.0.2" -psycopg2 = "^2.9.9" + +# Psycopg3 is not supported in SQLAlchemy 1.x: +# psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} + [tool.poetry.group.dev.dependencies] docker = "^7.0.0" @@ -37,7 +40,6 @@ pyarrow-stubs = "^10.0.1.7" pytest = "^7.4.3" pytest-docker = "^2.0.1" pytest-mypy = "^0.10.3" -pytest-postgresql = "^5.0.0" ruff = "^0.1.11" types-jsonschema = "^4.20.0.0" google-cloud-secret-manager = "^2.17.0" @@ -107,9 +109,11 @@ select = [ "TD", # flake8-todos "TID", # flake8-tidy-imports "TRY", # tryceratops + "TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead. + "TRY003", # Disallow vanilla string passing. Prefer kwargs to the exception constructur. "UP", # pyupgrade "W", # pycodestyle (warnings) - "YTT" # flake8-2020 + "YTT", # flake8-2020 ] ignore = [ # For rules reference, see https://docs.astral.sh/ruff/rules/ @@ -128,7 +132,6 @@ ignore = [ "S", # flake8-bandit (noisy, security related) "TD002", # Require author for TODOs "TRIO", # flake8-trio (opinionated, noisy) - "TRY003", # Exceptions with too-long string descriptions # TODO: re-evaluate once we have our own exception classes "INP001", # Dir 'examples' is part of an implicit namespace package. Add an __init__.py. # TODO: Consider re-enabling these before release: @@ -138,7 +141,6 @@ ignore = [ "FIX002", # Allow "TODO:" until release (then switch to requiring links via TDO003) "PLW0603", # Using the global statement to update _cache is discouraged "TD003", # Require links for TODOs # TODO: Re-enable when we disable FIX002 - "TRY002", # TODO: When we have time to tackle exception management ("Create your own exception") ] fixable = ["ALL"] unfixable = [ @@ -154,6 +156,7 @@ force-sort-within-sections = false lines-after-imports = 2 known-first-party = ["airbyte_cdk", "airbyte_protocol"] known-local-folder = ["airbyte_lib"] +required-imports = ["from __future__ import annotations"] known-third-party = [] section-order = [ "future", diff --git a/airbyte-lib/tests/conftest.py b/airbyte-lib/tests/conftest.py index a52cd23a555560..caabe34cfed537 100644 --- a/airbyte-lib/tests/conftest.py +++ b/airbyte-lib/tests/conftest.py @@ -10,7 +10,7 @@ from airbyte_lib.caches.snowflake import SnowflakeCacheConfig import docker -import psycopg +import psycopg2 as psycopg import pytest from google.cloud import secretmanager from pytest_docker.plugin import get_docker_ip diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/metadata.yaml b/airbyte-lib/tests/integration_tests/fixtures/source-broken/metadata.yaml new file mode 100644 index 00000000000000..f6585ba25c1b43 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/metadata.yaml @@ -0,0 +1,13 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 + dockerImageTag: 0.0.0 + dockerRepository: airbyte/source-broken + githubIssueLabel: source-broken + name: Test + releaseDate: 2023-08-25 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset +metadataSpecVersion: "1.0" diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py new file mode 100644 index 00000000000000..1172b397f49347 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +setup( + name="source_broken", + version="0.0.1", + description="Test Soutce", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + entry_points={ + "console_scripts": [ + "source-broken=source_broken.run:run", + ], + }, +) diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/source_broken/run.py b/airbyte-lib/tests/integration_tests/fixtures/source-broken/source_broken/run.py new file mode 100644 index 00000000000000..c777271f249a67 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/source_broken/run.py @@ -0,0 +1,4 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +def run(): + raise Exception("Could not run") \ No newline at end of file diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py index c9bdfbf91415ca..1ea3e13cd7cece 100644 --- a/airbyte-lib/tests/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -1,12 +1,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from collections.abc import Mapping import os import shutil -import subprocess +from typing import Any from unittest.mock import Mock, call, patch import tempfile from pathlib import Path +from sqlalchemy import column, text + import airbyte_lib as ab from airbyte_lib.caches import SnowflakeCacheConfig, SnowflakeSQLCache import pandas as pd @@ -16,6 +19,11 @@ from airbyte_lib.registry import _update_cache from airbyte_lib.version import get_version from airbyte_lib.results import ReadResult +from airbyte_lib.datasets import CachedDataset, LazyDataset, SQLDataset +import airbyte_lib as ab + +from airbyte_lib.results import ReadResult +from airbyte_lib import exceptions as exc @pytest.fixture(scope="module", autouse=True) @@ -185,6 +193,146 @@ def test_sync_with_merge_to_duckdb(expected_test_stream_data: dict[str, list[dic ) +def test_cached_dataset( + expected_test_stream_data: dict[str, list[dict[str, str | int]]], +) -> None: + source = ab.get_connector("source-test", config={"apiKey": "test"}) + result: ReadResult = source.read(ab.new_local_cache()) + + stream_name = "stream1" + not_a_stream_name = "not_a_stream" + + # Check that the stream appears in mapping-like attributes + assert stream_name in result.cache._streams_with_data + assert stream_name in result + assert stream_name in result.cache + assert stream_name in result.cache.streams + assert stream_name in result.streams + + stream_get_a: CachedDataset = result[stream_name] + stream_get_b: CachedDataset = result.streams[stream_name] + stream_get_c: CachedDataset = result.cache[stream_name] + stream_get_d: CachedDataset = result.cache.streams[stream_name] + + # Check that each get method is syntactically equivalent + + assert isinstance(stream_get_a, CachedDataset) + assert isinstance(stream_get_b, CachedDataset) + assert isinstance(stream_get_c, CachedDataset) + assert isinstance(stream_get_d, CachedDataset) + + assert stream_get_a == stream_get_b + assert stream_get_b == stream_get_c + assert stream_get_c == stream_get_d + + # Check that we can iterate over the stream + + list_from_iter_a = list(stream_get_a) + list_from_iter_b = [row for row in stream_get_a] + + # Make sure that we get a key error if we try to access a stream that doesn't exist + with pytest.raises(KeyError): + result[not_a_stream_name] + with pytest.raises(KeyError): + result.streams[not_a_stream_name] + with pytest.raises(KeyError): + result.cache[not_a_stream_name] + with pytest.raises(KeyError): + result.cache.streams[not_a_stream_name] + + # Make sure we can use "result.streams.items()" + for stream_name, cached_dataset in result.streams.items(): + assert isinstance(cached_dataset, CachedDataset) + assert isinstance(stream_name, str) + + list_data = list(cached_dataset) + assert list_data == expected_test_stream_data[stream_name] + + # Make sure we can use "result.cache.streams.items()" + for stream_name, cached_dataset in result.cache.streams.items(): + assert isinstance(cached_dataset, CachedDataset) + assert isinstance(stream_name, str) + + list_data = list(cached_dataset) + assert list_data == expected_test_stream_data[stream_name] + + +def test_cached_dataset_filter(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + result: ReadResult = source.read(ab.new_local_cache()) + + stream_name = "stream1" + + # Check the many ways to add a filter: + cached_dataset: CachedDataset = result[stream_name] + filtered_dataset_a: SQLDataset = cached_dataset.with_filter("column2 == 1") + filtered_dataset_b: SQLDataset = cached_dataset.with_filter(text("column2 == 1")) + filtered_dataset_c: SQLDataset = cached_dataset.with_filter(column("column2") == 1) + + assert isinstance(cached_dataset, CachedDataset) + all_records = list(cached_dataset) + assert len(all_records) == 2 + + for filtered_dataset, case in [ + (filtered_dataset_a, "a"), + (filtered_dataset_b, "b"), + (filtered_dataset_c, "c"), + ]: + assert isinstance(filtered_dataset, SQLDataset) + + # Check that we can iterate over each stream + + filtered_records: list[Mapping[str, Any]] = [row for row in filtered_dataset] + + # Check that the filter worked + assert len(filtered_records) == 1, f"Case '{case}' had incorrect number of records." + + # Assert the stream name still matches + assert filtered_dataset.stream_name == stream_name, \ + f"Case '{case}' had incorrect stream name." + + # Check that chaining filters works + chained_dataset = filtered_dataset.with_filter("column1 == 'value1'") + chained_records = [row for row in chained_dataset] + assert len(chained_records) == 1, \ + f"Case '{case}' had incorrect number of records after chaining filters." + + +def test_lazy_dataset_from_source( + expected_test_stream_data: dict[str, list[dict[str, str | int]]], +) -> None: + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + stream_name = "stream1" + not_a_stream_name = "not_a_stream" + + lazy_dataset_a = source.get_records(stream_name) + lazy_dataset_b = source.get_records(stream_name) + + assert isinstance(lazy_dataset_a, LazyDataset) + + # Check that we can iterate over the stream + + list_from_iter_a = list(lazy_dataset_a) + list_from_iter_b = [row for row in lazy_dataset_b] + + assert list_from_iter_a == list_from_iter_b + + # Make sure that we get a key error if we try to access a stream that doesn't exist + with pytest.raises(exc.AirbyteLibInputError): + source.get_records(not_a_stream_name) + + # Make sure we can iterate on all available streams + for stream_name in source.get_available_streams(): + assert isinstance(stream_name, str) + + lazy_dataset: LazyDataset = source.get_records(stream_name) + assert isinstance(lazy_dataset, LazyDataset) + + list_data = list(lazy_dataset) + assert list_data == expected_test_stream_data[stream_name] + + @pytest.mark.parametrize( "method_call", [ @@ -196,7 +344,7 @@ def test_sync_with_merge_to_duckdb(expected_test_stream_data: dict[str, list[dic def test_check_fail_on_missing_config(method_call): source = ab.get_connector("source-test") - with pytest.raises(Exception, match="Config is not set, either set in get_connector or via source.set_config"): + with pytest.raises(exc.AirbyteConnectorConfigurationMissingError): method_call(source) def test_sync_with_merge_to_postgres(new_pg_cache_config: PostgresCacheConfig, expected_test_stream_data: dict[str, list[dict[str, str | int]]]): @@ -260,7 +408,7 @@ def test_tracking(mock_datetime: Mock, mock_requests: Mock, raises: bool, api_ke source.read(cache) else: source.read(cache) - + mock_post.assert_has_calls([ call("https://api.segment.io/v1/track", @@ -387,4 +535,4 @@ def test_install_uninstall(): source.install() - source.check() \ No newline at end of file + source.check() diff --git a/airbyte-lib/tests/integration_tests/test_validation.py b/airbyte-lib/tests/integration_tests/test_validation.py index 75a463592833e0..69216232b28c78 100644 --- a/airbyte-lib/tests/integration_tests/test_validation.py +++ b/airbyte-lib/tests/integration_tests/test_validation.py @@ -8,8 +8,15 @@ def test_validate_success(): - validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/valid_config.json") + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/valid_config.json", validate_install_only=False) -def test_validate_failure(): +def test_validate_check_failure(): with pytest.raises(Exception): - validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json") + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json", validate_install_only=False) + +def test_validate_success_install_only(): + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json", validate_install_only=True) + +def test_validate_config_failure(): + with pytest.raises(Exception): + validate("./tests/integration_tests/fixtures/source-broken", "./tests/integration_tests/fixtures/valid_config.json", validate_install_only=True) diff --git a/airbyte-lib/tests/unit_tests/test_exceptions.py b/airbyte-lib/tests/unit_tests/test_exceptions.py new file mode 100644 index 00000000000000..ef5a391e47df0d --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_exceptions.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import inspect +import pytest +import inspect +import airbyte_lib.exceptions as exceptions_module + +def test_exceptions(): + exception_classes = [ + (name, obj) + for name, obj in inspect.getmembers(exceptions_module) + if inspect.isclass(obj) and name.endswith("Error") + ] + assert "AirbyteError" in [name for name, _ in exception_classes] + assert "NotAnError" not in [name for name, _ in exception_classes] + for name, obj in exception_classes: + instance = obj() + message = instance.get_message() + assert isinstance(message, str), "No message for class: " + name + assert message.count("\n") == 0 + assert message != "" + assert message.strip() == message + assert name.startswith("Airbyte") + assert name.endswith("Error") + + +if __name__ == "__main__": + pytest.main() diff --git a/buildSrc/src/main/groovy/airbyte-java-connector.gradle b/buildSrc/src/main/groovy/airbyte-java-connector.gradle index 9d8a60ed88c855..fd2064733568bb 100644 --- a/buildSrc/src/main/groovy/airbyte-java-connector.gradle +++ b/buildSrc/src/main/groovy/airbyte-java-connector.gradle @@ -47,6 +47,13 @@ class AirbyteJavaConnectorExtension { void addCdkDependencies() { def projectName = { ":airbyte-cdk:java:airbyte-cdk:${it}" } def jarName = { "io.airbyte.cdk:airbyte-cdk-${it}:${cdkVersionRequired}" } + project.processIntegrationTestJavaResources { + // The metadata.yaml file is required by DestinationAcceptanceTest. + from(project.projectDir) { + include 'metadata.yaml' + duplicatesStrategy DuplicatesStrategy.EXCLUDE + } + } project.dependencies { def dep = { useLocalCdk ? project.project(projectName(it)) : jarName(it) } def testFixturesDep = { useLocalCdk ? testFixtures(project.project(projectName(it))) : "${jarName(it)}:test-fixtures" } diff --git a/deps.toml b/deps.toml index fa3943bebe30dc..7c1f15650f7266 100644 --- a/deps.toml +++ b/deps.toml @@ -13,11 +13,6 @@ junit-jupiter = "5.9.1" kotlin = "1.9.0" log4j = "2.21.1" lombok = "1.18.24" -micronaut = "3.8.3" -micronaut-data = "3.9.4" -micronaut-jaxrs = "3.4.0" -micronaut-security = "3.9.2" -micronaut-test = "3.8.0" postgresql = "42.6.0" reactor = "3.5.2" segment = "2.1.1" @@ -117,45 +112,12 @@ debezium-mongodb = { module = "io.debezium:debezium-connector-mongodb", version. debezium-mysql = { module = "io.debezium:debezium-connector-mysql", version.ref = "debezium"} debezium-postgres = { module = "io.debezium:debezium-connector-postgres", version.ref = "debezium"} -# Micronaut-related dependencies -h2-database = { module = "com.h2database:h2", version = "2.1.214" } -hibernate-types = { module = "com.vladmihalcea:hibernate-types-52", version = "2.16.3" } -jakarta-inject = { module = "jakarta.annotation:jakarta.annotation-api", version = "2.1.1" } -javax-transaction = { module = "javax.transaction:javax.transaction-api", version = "1.3" } -micronaut-bom = { module = "io.micronaut:micronaut-bom", version.ref = "micronaut" } -micronaut-cache-caffeine = { module = "io.micronaut.cache:micronaut-cache-caffeine", version = "3.5.0"} -micronaut-data-processor = { module = "io.micronaut.data:micronaut-data-processor", version.ref = "micronaut-data" } -micronaut-data-tx = { module = "io.micronaut.data:micronaut-data-tx", version.ref = "micronaut-data" } -micronaut-flyway = { module = "io.micronaut.flyway:micronaut-flyway", version = "5.4.1" } -micronaut-inject = { module = "io.micronaut:micronaut-inject" } -micronaut-http = { module = "io.micronaut:micronaut-http", version.ref = "micronaut" } -micronaut-http-client = { module = "io.micronaut:micronaut-http-client" } -micronaut-http-server-netty = { module = "io.micronaut:micronaut-http-server-netty", version.ref = "micronaut" } -micronaut-inject-java = { module = "io.micronaut:micronaut-inject-java", version.ref = "micronaut" } -micronaut-jaxrs-processor = { module = "io.micronaut.jaxrs:micronaut-jaxrs-processor", version.ref = "micronaut-jaxrs" } -micronaut-jaxrs-server = { module = "io.micronaut.jaxrs:micronaut-jaxrs-server", version.ref = "micronaut-jaxrs" } -micronaut-jdbc = { module = "io.micronaut.sql:micronaut-jdbc", version = "4.7.2" } -micronaut-jdbc-hikari = { module = "io.micronaut.sql:micronaut-jdbc-hikari" } -micronaut-jooq = { module = "io.micronaut.sql:micronaut-jooq" } -micronaut-management = { module = "io.micronaut:micronaut-management" } -micronaut-runtime = { module = "io.micronaut:micronaut-runtime" } -micronaut-security = { module = "io.micronaut.security:micronaut-security", version.ref = "micronaut-security" } -micronaut-test-core = { module = "io.micronaut.test:micronaut-test-core", version.ref = "micronaut-test" } -micronaut-test-junit5 = { module = "io.micronaut.test:micronaut-test-junit5", version.ref = "micronaut-test" } -micronaut-validation = { module = "io.micronaut:micronaut-validation" } - [bundles] apache = ["apache-commons", "apache-commons-lang"] datadog = ["datadog-trace-api", "datadog-trace-ot"] jackson = ["jackson-databind", "jackson-annotations", "jackson-dataformat", "jackson-datatype"] junit = ["junit-jupiter-api", "junit-jupiter-params", "mockito-junit-jupiter"] log4j = ["log4j-api", "log4j-core", "log4j-impl", "log4j-web"] -micronaut = ["jakarta-inject", "javax-transaction", "micronaut-http-server-netty", "micronaut-http-client", "micronaut-inject", "micronaut-validation", "micronaut-runtime", "micronaut-management", "micronaut-security", "micronaut-jaxrs-server", "micronaut-flyway", "micronaut-jdbc-hikari", "micronaut-jooq"] -micronaut-annotation = ["jakarta-inject", "micronaut-inject-java"] -micronaut-annotation-processor = ["micronaut-inject-java", "micronaut-management", "micronaut-validation", "micronaut-data-processor", "micronaut-jaxrs-processor"] -micronaut-server = ["micronaut-jaxrs-processor", "micronaut-jaxrs-server"] -micronaut-test = ["micronaut-test-core", "micronaut-test-junit5", "h2-database"] -micronaut-test-annotation-processor = ["micronaut-inject-java"] slf4j = ["jul-to-slf4j", "jcl-over-slf4j", "log4j-over-slf4j"] temporal = ["temporal-sdk", "temporal-serviceclient"] diff --git a/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md b/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md index a98dd3beac865b..f5822a2d28eda4 100644 --- a/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md +++ b/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md @@ -29,7 +29,7 @@ To set up the dbt Cloud integration in Airbyte Cloud: 1. In the Airbyte UI, click **Settings**. -2. Click **dbt Cloud integration**. +2. Click **Integrations**. 3. Paste the service token from [Step 1](#step-1-generate-a-service-token) and click **Save changes**. diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index d33f9d148df52d..34795a73856c2a 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -120,7 +120,7 @@ The steps for updating an existing connector are the same as for building a new _Coming soon._ -Typing and Deduplication is how Airbyte transforms the raw data which is transmitted during a sync into easy-to-use final tables for database and data warehouse destinations. For more information on how typing and deduplication works, see [this doc](/understanding-airbyte/typing-deduping). +Typing and Deduplication is how Airbyte transforms the raw data which is transmitted during a sync into easy-to-use final tables for database and data warehouse destinations. For more information on how typing and deduplication works, see [this doc](/using-airbyte/core-concepts/typing-deduping). ## Publishing a connector diff --git a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md index 7d650d089415cf..e66c4cecd92751 100644 --- a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md @@ -144,12 +144,15 @@ Additional tests are validating the backward compatibility of the current specif These backward compatibility tests can be bypassed by changing the value of the `backward_compatibility_tests_config.disable_for_version` input in `acceptance-test-config.yml` (see below). One more test validates the specification against containing exposed secrets. This means fields that potentially could hold a secret value should be explicitly marked with `"airbyte_secret": true`. If an input field like `api_key` / `password` / `client_secret` / etc. is exposed, the test will fail. -| Input | Type | Default | Note | -| :--------------------------------------------------------------- | :----- | :------------------ | :-------------------------------------------------------------------------------------------------------------------- | -| `spec_path` | string | `secrets/spec.json` | Path to a YAML or JSON file representing the spec expected to be output by this connector | -| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests (expects a version following semantic versioning). | -| `backward_compatibility_tests_config.disable_for_version` | string | None | Disable the backward compatibility test for a specific version (expects a version following semantic versioning). | -| `timeout_seconds` | int | 10 | Test execution timeout in seconds | +| Input | Type | Default | Note | +|:-----------------------------------------------------------------|:--------|:--------------------|:----------------------------------------------------------------------------------------------------------------------| +| `spec_path` | string | `secrets/spec.json` | Path to a YAML or JSON file representing the spec expected to be output by this connector | +| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests (expects a version following semantic versioning). | +| `backward_compatibility_tests_config.disable_for_version` | string | None | Disable the backward compatibility test for a specific version (expects a version following semantic versioning). | +| `timeout_seconds` | int | 10 | Test execution timeout in seconds | +| `auth_default_method` | object | None | Ensure that OAuth is default method, if OAuth uses by source | +| `auth_default_method.oauth` | boolean | True | Validate that OAuth is default method if set to True | +| `auth_default_method.bypass_reason` | string | | Reason why OAuth is not default method | ## Test Connection @@ -180,26 +183,32 @@ These backward compatibility tests can be bypassed by changing the value of the Configuring all streams in the input catalog to full refresh mode verifies that a read operation produces some RECORD messages. Each stream should have some data, if you can't guarantee this for particular streams - add them to the `empty_streams` list. Set `validate_data_points=True` if possible. This validation is going to be enabled by default and won't be configurable in future releases. -| Input | Type | Default | Note | -| :---------------------------------------- | :--------------- | :------------------------------------------ | :------------------------------------------------------------------------------------------------------------ | -| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | -| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | -| `empty_streams` | array of objects | \[\] | List of streams that might be empty with a `bypass_reason` | -| `empty_streams[0].name` | string | | Name of the empty stream | -| `empty_streams[0].bypass_reason` | string | None | Reason why this stream is empty | -| `ignored_fields[stream][0].name` | string | | Name of the ignored field | -| `ignored_fields[stream][0].bypass_reason` | string | None | Reason why this field is ignored | -| `validate_schema` | boolean | True | Verify that structure and types of records matches the schema from discovery command | -| `fail_on_extra_columns` | boolean | True | Fail schema validation if undeclared columns are found in records. Only relevant when `validate_schema=True` | -| `validate_data_points` | boolean | False | Validate that all fields in all streams contained at least one data point | -| `timeout_seconds` | int | 5\*60 | Test execution timeout in seconds | -| `expect_trace_message_on_failure` | boolean | True | Ensure that a trace message is emitted when the connector crashes | -| `expect_records` | object | None | Compare produced records with expected records, see details below | -| `expect_records.path` | string | | File with expected records | -| `expect_records.bypass_reason` | string | | Explain why this test is bypassed | -| `expect_records.extra_fields` | boolean | False | Allow output records to have other fields i.e: expected records are a subset | -| `expect_records.exact_order` | boolean | False | Ensure that records produced in exact same order | -| `expect_records.extra_records` | boolean | True | Allow connector to produce extra records, but still enforce all records from the expected file to be produced | +| Input | Type | Default | Note | +|:------------------------------------------------|:-----------------|:--------------------------------------------|:--------------------------------------------------------------------------------------------------------------| +| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | +| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | +| `empty_streams` | array of objects | \[\] | List of streams that might be empty with a `bypass_reason` | +| `empty_streams[0].name` | string | | Name of the empty stream | +| `empty_streams[0].bypass_reason` | string | None | Reason why this stream is empty | +| `ignored_fields[stream][0].name` | string | | Name of the ignored field | +| `ignored_fields[stream][0].bypass_reason` | string | None | Reason why this field is ignored | +| `validate_schema` | boolean | True | Verify that structure and types of records matches the schema from discovery command | +| `fail_on_extra_columns` | boolean | True | Fail schema validation if undeclared columns are found in records. Only relevant when `validate_schema=True` | +| `validate_data_points` | boolean | False | Validate that all fields in all streams contained at least one data point | +| `timeout_seconds` | int | 5\*60 | Test execution timeout in seconds | +| `expect_trace_message_on_failure` | boolean | True | Ensure that a trace message is emitted when the connector crashes | +| `expect_records` | object | None | Compare produced records with expected records, see details below | +| `expect_records.path` | string | | File with expected records | +| `expect_records.bypass_reason` | string | | Explain why this test is bypassed | +| `expect_records.extra_fields` | boolean | False | Allow output records to have other fields i.e: expected records are a subset | +| `expect_records.exact_order` | boolean | False | Ensure that records produced in exact same order | +| `expect_records.extra_records` | boolean | True | Allow connector to produce extra records, but still enforce all records from the expected file to be produced | +| `file_types` | object | None | Configure file-based connectors specific tests | +| `file_types.skip_test` | boolean | False | Skip file-based connectors specific tests for the current config with a `bypass_reason` | +| `file_types.bypass_reason` | string | None | Reason why file-based connectors specific tests are skipped | +| `file_types.unsupported_types` | array of objects | None | Configure file types which are not supported by a source | +| `file_types.unsupported_types[0].extension` | string | | File type in `.csv` format which cannot be added to a test account | +| `file_types.unsupported_types[0].bypass_reason` | string | None | Reason why this file type cannot be added to a test account | `expect_records` is a nested configuration, if omitted - the part of the test responsible for record matching will be skipped. Due to the fact that we can't identify records without primary keys, only the following flag combinations are supported: @@ -289,6 +298,10 @@ Some examples of the types of tests covered are verification that streams define |:------------------------------------------|:-----------------|:----------------------|:-----------------------------------------------------------------------| | `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | | `streams_without_primary_key` | array of objects | None | List of streams that do not support a primary key like reports streams | +| `streams_without_primary_key.name` | string | None | Name of the stream missing the PK | +| `streams_without_primary_key.bypass_reason` | string | None | The reason the stream doesn't have the PK | +| `allowed_hosts.bypass_reason` | object with `bypass_reason` | None | Defines the `bypass_reason` description about why the `allowedHosts` check for the certified connector should be skipped | +| `suggested_streams.bypass_reason` | object with `bypass_reason` | None | Defines the `bypass_reason` description about why the `suggestedStreams` check for the certified connector should be skipped | ## Strictness level diff --git a/docs/deploying-airbyte/on-kubernetes-via-helm.md b/docs/deploying-airbyte/on-kubernetes-via-helm.md index 375a59b8576d88..060e5a6e66545b 100644 --- a/docs/deploying-airbyte/on-kubernetes-via-helm.md +++ b/docs/deploying-airbyte/on-kubernetes-via-helm.md @@ -2,7 +2,7 @@ ## Overview -Airbyte allows scaling sync workloads horizontally using Kubernetes. The core components \(api server, scheduler, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. +Airbyte allows scaling sync workloads horizontally using Kubernetes. The core components \(api server, worker, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. ## Quickstart @@ -14,6 +14,11 @@ Alternatively, you can deploy Airbyte on [Restack](https://www.restack.io) to pr Airbyte running on Self-Hosted Kubernetes doesn't support DBT Transformations. Please refer to [#5901](https://github.com/airbytehq/airbyte/issues/5091) ::: +:::note +Airbyte Kubernetes Community Edition does not support basic auth by default. +To enable basic auth, consider adding a reverse proxy in front of Airbyte. +::: + ## Getting Started ### Cluster Setup diff --git a/docs/enterprise-setup/assets/self-managed-enterprise-aws.png b/docs/enterprise-setup/assets/self-managed-enterprise-aws.png new file mode 100644 index 00000000000000..5bb6c7d92ac249 Binary files /dev/null and b/docs/enterprise-setup/assets/self-managed-enterprise-aws.png differ diff --git a/docs/enterprise-setup/implementation-guide.md b/docs/enterprise-setup/implementation-guide.md index a12843e3c19e2a..e01a4123d6743c 100644 --- a/docs/enterprise-setup/implementation-guide.md +++ b/docs/enterprise-setup/implementation-guide.md @@ -13,33 +13,50 @@ Airbyte Self-Managed Enterprise must be deployed using Kubernetes. This is to en ## Prerequisites -There are three prerequisites to deploying: installing [helm](https://helm.sh/docs/intro/install/), a Kubernetes cluster, and having configured `kubectl` to connect to the cluster. +For a production-ready deployment of Self-Managed Enterprise, various infrastructure components are required. We recommend deploying to Amazon EKS or Google Kubernetes Engine. The following diagram illustrates a typical Airbyte deployment running on AWS: -For production, we recommend deploying to EKS, GKE or AKS. If you are doing some local testing, follow the cluster setup instructions outlined [here](/deploying-airbyte/on-kubernetes-via-helm.md#cluster-setup). +![AWS Architecture Diagram](./assets/self-managed-enterprise-aws.png) -To install `kubectl`, please follow [these instructions](https://kubernetes.io/docs/tasks/tools/). To configure `kubectl` to connect to your cluster by using `kubectl use-context my-cluster-name`, see the following: +Prior to deploying Self-Managed Enterprise, we recommend having each of the following infrastructure components ready to go. When possible, it's easiest to have all components running in the same [VPC](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). The provided recommendations are for customers deploying to AWS: + +| Component | Recommendation | +|--------------------------|-----------------------------------------------------------------------------| +| Kubernetes Cluster | Amazon EKS cluster running in [2 or more availability zones](https://docs.aws.amazon.com/eks/latest/userguide/disaster-recovery-resiliency.html) on a minimum of 6 nodes. | +| Ingress | [Amazon ALB](#configuring-ingress) and a URL for users to access the Airbyte UI or make API requests. | +| Object Storage | [Amazon S3 bucket](#configuring-external-logging) with two directories for log and state storage. | +| Dedicated Database | [Amazon RDS Postgres](#configuring-the-airbyte-database) with at least one read replica. | +| External Secrets Manager | [Amazon Secrets Manager](/operator-guides/configuring-airbyte#secrets) for storing connector secrets. | + + +We also require you to install and configure the following Kubernetes tooling: +1. Install `helm` by following [these instructions](https://helm.sh/docs/intro/install/) +2. Install `kubectl` by following [these instructions](https://kubernetes.io/docs/tasks/tools/). +3. Configure `kubectl` to connect to your cluster by using `kubectl use-context my-cluster-name`:
- Configure kubectl to connect to your cluster - - -
    -
  1. Configure gcloud with gcloud auth login.
    2. -
  2. On the Google Cloud Console, the cluster page will have a "Connect" button, with a command to run locally: gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME
    3. -
  3. Use kubectl config get-contexts to show the contexts available.
    4. -
  4. Run kubectl config use-context $GKE_CONTEXT to access the cluster from kubectl.
    5. -
- - -
    -
  1. Configure your AWS CLI to connect to your project.
    2. -
  2. Install eksctl.
    3. -
  3. Run eksctl utils write-kubeconfig --cluster=$CLUSTER_NAME to make the context available to kubectl.
    4. -
  4. Use kubectl config get-contexts to show the contexts available.
    5. -
  5. Run kubectl config use-context $EKS_CONTEXT to access the cluster with kubectl.
    6. -
- - +Configure kubectl to connect to your cluster + + + + +1. Configure your [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to connect to your project. +2. Install [eksctl](https://eksctl.io/introduction/). +3. Run `eksctl utils write-kubeconfig --cluster=$CLUSTER_NAME` to make the context available to kubectl. +4. Use `kubectl config get-contexts` to show the available contexts. +5. Run `kubectl config use-context $EKS_CONTEXT` to access the cluster with kubectl. + + + + + +1. Configure `gcloud` with `gcloud auth login`. +2. On the Google Cloud Console, the cluster page will have a "Connect" button, with a command to run locally: `gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME`. +3. Use `kubectl config get-contexts` to show the available contexts. +4. Run `kubectl config use-context $EKS_CONTEXT` to access the cluster with kubectl. + + + +
## Deploy Airbyte Enterprise diff --git a/docs/integrations/destinations/astra.md b/docs/integrations/destinations/astra.md new file mode 100644 index 00000000000000..55436858830764 --- /dev/null +++ b/docs/integrations/destinations/astra.md @@ -0,0 +1,41 @@ +# Astra Destination + +This page contains the setup guide and reference information for the destination-astra connector. + +## Pre-Requisites + +- An OpenAI, AzureOpenAI, Cohere, etc. API Key + +## Setup Guide + +#### Set Up an Astra Database + +- Create an Astra account [here](https://astra.datastax.com/signup) +- In the Astra Portal, select Databases in the main navigation. +- Click Create Database. +- In the Create Database dialog, select the Serverless (Vector) deployment type. +- In the Configuration section, enter a name for the new database in the Database name field. +-- Because database names can’t be changed later, it’s best to name your database something meaningful. Database names must start and end with an alphanumeric character, and may contain the following special characters: & + - _ ( ) < > . , @. +- Select your preferred Provider and Region. +-- You can select from a limited number of regions if you’re on the Free plan. Regions with a lock icon require that you upgrade to a Pay As You Go plan. +- Click Create Database. +-- You are redirected to your new database’s Overview screen. Your database starts in Pending status before transitioning to Initializing. You’ll receive a notification once your database is initialized. + +#### Gathering other credentials + +- Go back to the Overview tab on the Astra UI +- Copy the Endpoint under Database Details and load into Airbyte under the name astra_db_endpoint +- Click generate token, copy the application token and load under astra_db_app_token + +## Supported Sync Modes + +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | Yes | | + +## Changelog +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| 0.1.0 | 2024-01-08 | | Initial Release | diff --git a/docs/integrations/destinations/bigquery-migrations.md b/docs/integrations/destinations/bigquery-migrations.md index 7f62d4880b4cff..059044e8cec99c 100644 --- a/docs/integrations/destinations/bigquery-migrations.md +++ b/docs/integrations/destinations/bigquery-migrations.md @@ -11,4 +11,4 @@ Worthy of specific mention, this version includes: - Removal of sub-tables for nested properties - Removal of SCD tables -Learn more about what's new in Destinations V2 [here](/understanding-airbyte/typing-deduping). \ No newline at end of file +Learn more about what's new in Destinations V2 [here](/using-airbyte/core-concepts/typing-deduping). \ No newline at end of file diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index b30ba594a0cb85..fd89f707091d88 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -143,7 +143,7 @@ could be subject to change in future versions. misformatted or unexpected data. The column type in BigQuery is `JSON`. ... and a column of the proper data type for each of the top-level properties from your source's schema. Arrays and Objects will remain as JSON columns in BigQuery. Learn more about Typing and Deduping - [here](/understanding-airbyte/typing-deduping) + [here](/using-airbyte/core-concepts/typing-deduping) The output tables in BigQuery are partitioned by the Time-unit column `airbyte_extracted_at` at a daily granularity and clustered by `airbyte_extracted_at` and the table Primary Keys. Partitions @@ -210,10 +210,13 @@ tutorials: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2.3.31 | 2024-01-22 | [34023](https://github.com/airbytehq/airbyte/pull/34023) | Combine DDL operations into a single execution | -| 2.3.30 | 2024-01-12 | [34226](https://github.com/airbytehq/airbyte/pull/34226) | Upgrade CDK to 0.12.0; Cleanup dependencies | -| 2.3.29 | 2024-01-09 | [34003](https://github.com/airbytehq/airbyte/pull/34003) | Fix loading credentials from GCP Env | -| 2.3.28 | 2024-01-08 | [34021](https://github.com/airbytehq/airbyte/pull/34021) | Add idempotency ids in dummy insert for check call | +| 2.4.2 | 2024-01-24 | [34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | +| 2.4.1 | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | +| 2.4.0 | 2024-01-24 | [34468](https://github.com/airbytehq/airbyte/pull/34468) | Upgrade CDK to 0.14.0 | +| 2.3.31 | 2024-01-22 | [\#34023](https://github.com/airbytehq/airbyte/pull/34023) | Combine DDL operations into a single execution | +| 2.3.30 | 2024-01-12 | [\#34226](https://github.com/airbytehq/airbyte/pull/34226) | Upgrade CDK to 0.12.0; Cleanup dependencies | +| 2.3.29 | 2024-01-09 | [\#34003](https://github.com/airbytehq/airbyte/pull/34003) | Fix loading credentials from GCP Env | +| 2.3.28 | 2024-01-08 | [\#34021](https://github.com/airbytehq/airbyte/pull/34021) | Add idempotency ids in dummy insert for check call | | 2.3.27 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Skip retrieving initial table state when setup fails | | 2.3.26 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | | 2.3.25 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Update to java CDK 0.10.0 (no changes) | diff --git a/docs/integrations/destinations/redshift-migrations.md b/docs/integrations/destinations/redshift-migrations.md new file mode 100644 index 00000000000000..59d91b557f86e4 --- /dev/null +++ b/docs/integrations/destinations/redshift-migrations.md @@ -0,0 +1,14 @@ +# Redshift Migration Guide + +## Upgrading to 2.0.0 + +This version introduces [Destinations V2](/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. + +Worthy of specific mention, this version includes: + +- Per-record error handling +- Clearer table structure +- Removal of sub-tables for nested properties +- Removal of SCD tables + +Learn more about what's new in Destinations V2 [here](/using-airbyte/core-concepts/typing-deduping). diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 7a551b8ba69680..8f29d276dd89ce 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -9,7 +9,7 @@ The Airbyte Redshift destination allows you to sync data to Redshift. This Redshift destination connector has two replication strategies: 1. INSERT: Replicates data via SQL INSERT queries. This is built on top of the destination-jdbc code - base and is configured to rely on JDBC 4.2 standard drivers provided by Amazon via Mulesoft + base and is configured to rely on JDBC 4.2 standard drivers provided by Amazon via Maven Central [here](https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42) as described in Redshift documentation [here](https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-install.html). **Not recommended @@ -28,8 +28,8 @@ For INSERT strategy: 2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is the recommended loading approach described by Redshift - [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_loading-data-best-practices.html). - Requires an S3 bucket and credentials. + [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_best-practices-single-copy-command.html). + Requires an S3 bucket and credentials. Data is copied into S3 as multiple files with a manifest file. Airbyte automatically picks an approach depending on the given configuration - if S3 configuration is present, Airbyte will use the COPY strategy and vice versa. @@ -76,8 +76,9 @@ Optional parameters: (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after the `COPY` command completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`. -NOTE: S3 staging does not use the SSH Tunnel option, if configured. SSH Tunnel supports the SQL -connection only. S3 is secured through public HTTPS access only. +NOTE: S3 staging does not use the SSH Tunnel option for copying data, if configured. SSH Tunnel supports the SQL +connection only. S3 is secured through public HTTPS access only. Subsequent typing and deduping queries on final table +are executed over using provided SSH Tunnel configuration. ## Step 1: Set up Redshift @@ -98,10 +99,10 @@ connection only. S3 is secured through public HTTPS access only. ### Permissions in Redshift Airbyte writes data into two schemas, whichever schema you want your data to land in, e.g. `my_schema` and a "Raw Data" schema that Airbyte uses to improve ELT reliability. By default, this raw data schema -is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings. +is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings. Airbyte also needs to query Redshift's -[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for -metadata about the tables airbyte manages. +[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for +metadata about the tables airbyte manages. To ensure the `airbyte_user` has the correction permissions to: - create schemas in your database @@ -204,32 +205,44 @@ All Redshift connections are encrypted using SSL. Each stream will be output into its own raw table in Redshift. Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in +- `_airbyte_raw_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Redshift is `VARCHAR`. -- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. +- `_airbyte_extracted_at`: a timestamp representing when the event was pulled from the data source. The column type in Redshift is `TIMESTAMP WITH TIME ZONE`. +- `_airbyte_loaded_at`: a timestamp representing when the row was processed into final table. + The column type in Redshift is `TIMESTAMP WITH TIME ZONE`. - `_airbyte_data`: a json blob representing with the event data. The column type in Redshift is `SUPER`. -## Data type mapping - -| Redshift Type | Airbyte Type | Notes | -| :-------------------- | :------------------------ | :---- | -| `boolean` | `boolean` | | -| `int` | `integer` | | -| `float` | `number` | | -| `varchar` | `string` | | -| `date/varchar` | `date` | | -| `time/varchar` | `time` | | -| `timestamptz/varchar` | `timestamp_with_timezone` | | -| `varchar` | `array` | | -| `varchar` | `object` | | +## Data type map + +| Airbyte type | Redshift type | +|:------------------------------------|:---------------------------------------| +| STRING | VARCHAR | +| STRING (BASE64) | VARCHAR | +| STRING (BIG_NUMBER) | VARCHAR | +| STRING (BIG_INTEGER) | VARCHAR | +| NUMBER | DECIMAL / NUMERIC | +| INTEGER | BIGINT / INT8 | +| BOOLEAN | BOOLEAN / BOOL | +| STRING (TIMESTAMP_WITH_TIMEZONE) | TIMESTAMPTZ / TIMESTAMP WITH TIME ZONE | +| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | TIMESTAMP | +| STRING (TIME_WITH_TIMEZONE) | TIMETZ / TIME WITH TIME ZONE | +| STRING (TIME_WITHOUT_TIMEZONE) | TIME | +| DATE | DATE | +| OBJECT | SUPER | +| ARRAY | SUPER | ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.8.0 | 2024-01-18 | [34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0 | +| 2.1.3 | 2024-01-26 | [34544](https://github.com/airbytehq/airbyte/pull/34544) | Proper string-escaping in raw tables | +| 2.1.2 | 2024-01-24 | [34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | +| 2.1.1 | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | +| 2.1.0 | 2024-01-24 | [34467](https://github.com/airbytehq/airbyte/pull/34467) | Upgrade CDK to 0.14.0 | +| 2.0.0 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Destinations V2 | +| 0.8.0 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0 | | 0.7.15 | 2024-01-11 | [\#34186](https://github.com/airbytehq/airbyte/pull/34186) | Update check method with svv_table_info permission check, fix bug where s3 staging files were not being deleted. | | 0.7.14 | 2024-01-08 | [\#34014](https://github.com/airbytehq/airbyte/pull/34014) | Update order of options in spec | | 0.7.13 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Fix NPE when prepare tables fail; Add case sensitive session for super; Bastion heartbeats added | @@ -308,4 +321,4 @@ Each stream will be output into its own raw table in Redshift. Each table will c | 0.3.14 | 2021-10-08 | [\#5924](https://github.com/airbytehq/airbyte/pull/5924) | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table | | 0.3.13 | 2021-09-02 | [\#5745](https://github.com/airbytehq/airbyte/pull/5745) | Disable STATUPDATE flag when using S3 staging to speed up performance | | 0.3.12 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Enable partial checkpointing for halfway syncs | -| 0.3.11 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | \ No newline at end of file +| 0.3.11 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | diff --git a/docs/integrations/destinations/snowflake-migrations.md b/docs/integrations/destinations/snowflake-migrations.md index 2da5fe727be878..adb75e5126e9e7 100644 --- a/docs/integrations/destinations/snowflake-migrations.md +++ b/docs/integrations/destinations/snowflake-migrations.md @@ -11,7 +11,7 @@ Worthy of specific mention, this version includes: - Removal of sub-tables for nested properties - Removal of SCD tables -Learn more about what's new in Destinations V2 [here](/understanding-airbyte/typing-deduping). +Learn more about what's new in Destinations V2 [here](/using-airbyte/core-concepts/typing-deduping). ## Upgrading to 2.0.0 diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index aad1612b10c828..d597cb70d66cf8 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -246,8 +246,13 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.4.22 | 2024-01-12 | [34227](https://github.com/airbytehq/airbyte/pull/34227) | Upgrade CDK to 0.12.0; Cleanup unused dependencies | -| 3.4.21 | 2024-01-10 | [\#34083](https://github.com/airbytehq/airbte/pull/34083) | Emit destination stats as part of the state message | +| 3.5.4 | 2024-01-24 | [\#34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | +| 3.5.3 | 2024-01-25 | [\#34528](https://github.com/airbytehq/airbyte/pull/34528) | Fix spurious `check` failure (`UnsupportedOperationException: Snowflake does not use the native JDBC DV2 interface`) | +| 3.5.2 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | +| 3.5.1 | 2024-01-24 | [\#34501](https://github.com/airbytehq/airbyte/pull/34501) | Internal code changes for Destinations V2 | +| 3.5.0 | 2024-01-24 | [\#34462](https://github.com/airbytehq/airbyte/pull/34462) | Upgrade CDK to 0.14.0 | +| 3.4.22 | 2024-01-12 | [\#34227](https://github.com/airbytehq/airbyte/pull/34227) | Upgrade CDK to 0.12.0; Cleanup unused dependencies | +| 3.4.21 | 2024-01-10 | [\#34083](https://github.com/airbytehq/airbyte/pull/34083) | Emit destination stats as part of the state message | | 3.4.20 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Skip retrieving initial table state when setup fails | | 3.4.19 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | | 3.4.18 | 2024-01-02 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Add option to only type and dedupe at the end of the sync | @@ -403,4 +408,4 @@ Otherwise, make sure to grant the role the required permissions in the desired n | 0.3.13 | 2021-09-01 | [\#5784](https://github.com/airbytehq/airbyte/pull/5784) | Updated query timeout from 30 minutes to 3 hours | | 0.3.12 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | | 0.3.11 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.3.10 | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713) | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake | \ No newline at end of file +| 0.3.10 | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713) | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake | diff --git a/docs/integrations/destinations/weaviate.md b/docs/integrations/destinations/weaviate.md index c8acb02b50de36..583247263ac029 100644 --- a/docs/integrations/destinations/weaviate.md +++ b/docs/integrations/destinations/weaviate.md @@ -85,6 +85,7 @@ When using [multi-tenancy](https://weaviate.io/developers/weaviate/manage-data/m | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------- | +| 0.2.15 | 2023-01-25 | [34529](https://github.com/airbytehq/airbyte/pull/34529) | Fix tests | | 0.2.14 | 2023-01-15 | [34229](https://github.com/airbytehq/airbyte/pull/34229) | Allow configuring tenant id | | 0.2.13 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | | 0.2.12 | 2023-12-07 | [33218](https://github.com/airbytehq/airbyte/pull/33218) | Normalize metadata field names | diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index 1dcb6d735fc0ce..5c93e1366c16ad 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -29,9 +29,120 @@ Use the service account ID from above, grant read access to your target bucket. ### Set up the source in Airbyte UI -- Paste the service account JSON key to `service_account` -- Enter your GCS bucket name to `gcs_bucket` -- Enter path to your file(s) to `gcs_path` +- Paste the service account JSON key to the `Service Account Information` field +- Enter your GCS bucket name to the `Bucket` field +- Add a stream + 1. Give a **Name** to the stream + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported format is **CSV**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). +- Configure the optional **Start Date** parameter that marks a starting date and time in UTC for data replication. Any files that have _not_ been modified since this specified date/time will _not_ be replicated. Use the provided datepicker (recommended) or enter the desired date programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. Leaving this field blank will replicate data from all files that have not been excluded by the **Path Pattern** and **Path Prefix**. +- Click **Set up source** and wait for the tests to complete. + +## Path Patterns + +\(tl;dr -> path pattern syntax using [wcmatch.glob](https://facelessuser.github.io/wcmatch/glob/). GLOBSTAR and SPLIT flags are enabled.\) + +This connector can sync multiple files by using glob-style patterns, rather than requiring a specific path for every file. This enables: + +- Referencing many files with just one pattern, e.g. `**` would indicate every file in the folder. +- Referencing future files that don't exist yet \(and therefore don't have a specific path\). + +You must provide a path pattern. You can also provide many patterns split with \| for more complex directory layouts. + +Each path pattern is a reference from the _root_ of the folder, so don't include the root folder name itself in the pattern\(s\). + +Some example patterns: + +- `**` : match everything. +- `**/*.csv` : match all files with specific extension. +- `myFolder/**/*.csv` : match all csv files anywhere under myFolder. +- `*/**` : match everything at least one folder deep. +- `*/*/*/**` : match everything at least three folders deep. +- `**/file.*|**/file` : match every file called "file" with any extension \(or no extension\). +- `x/*/y/*` : match all files that sit in sub-folder x -> any folder -> folder y. +- `**/prefix*.csv` : match all csv files with specific prefix. +- `**/prefix*.parquet` : match all parquet files with specific prefix. + +Let's look at a specific example, matching the following folder layout (`MyFolder` is the folder specified in the connector config as the root folder, which the patterns are relative to): + +```text +MyFolder + -> log_files + -> some_table_files + -> part1.csv + -> part2.csv + -> images + -> more_table_files + -> part3.csv + -> extras + -> misc + -> another_part1.csv +``` + +We want to pick up part1.csv, part2.csv and part3.csv \(excluding another_part1.csv for now\). We could do this a few different ways: + +- We could pick up every csv file called "partX" with the single pattern `**/part*.csv`. +- To be a bit more robust, we could use the dual pattern `some_table_files/*.csv|more_table_files/*.csv` to pick up relevant files only from those exact folders. +- We could achieve the above in a single pattern by using the pattern `*table_files/*.csv`. This could however cause problems in the future if new unexpected folders started being created. +- We can also recursively wildcard, so adding the pattern `extras/**/*.csv` would pick up any csv files nested in folders below "extras", such as "extras/misc/another_part1.csv". + +As you can probably tell, there are many ways to achieve the same goal with path patterns. We recommend using a pattern that ensures clarity and is robust against future additions to the directory structure. + +## User Schema + +When using the Avro, Jsonl, CSV or Parquet format, you can provide a schema to use for the output stream. **Note that this doesn't apply to the experimental Document file type format.** + +Providing a schema allows for more control over the output of this stream. Without a provided schema, columns and datatypes will be inferred from the first created file in the bucket matching your path pattern and suffix. This will probably be fine in most cases but there may be situations you want to enforce a schema instead, e.g.: + +- You only care about a specific known subset of the columns. The other columns would all still be included, but packed into the `_ab_additional_properties` map. +- Your initial dataset is quite small \(in terms of number of records\), and you think the automatic type inference from this sample might not be representative of the data in the future. +- You want to purposely define types for every column. +- You know the names of columns that will be added to future data and want to include these in the core schema as columns rather than have them appear in the `_ab_additional_properties` map. + +Or any other reason! The schema must be provided as valid JSON as a map of `{"column": "datatype"}` where each datatype is one of: + +- string +- number +- integer +- object +- array +- boolean +- null + +For example: + +- `{"id": "integer", "location": "string", "longitude": "number", "latitude": "number"}` +- `{"username": "string", "friends": "array", "information": "object"}` + +## File Format Settings + +### CSV + +Since CSV files are effectively plain text, providing specific reader options is often required for correct parsing of the files. These settings are applied when a CSV is created or exported so please ensure that this process happens consistently over time. + +- **Header Definition**: How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can set a value for the "Skip rows before header" option to ignore the header row. +- **Delimiter**: Even though CSV is an acronym for Comma Separated Values, it is used more generally as a term for flat file data that may or may not be comma separated. The delimiter field lets you specify which character acts as the separator. To use [tab-delimiters](https://en.wikipedia.org/wiki/Tab-separated_values), you can set this value to `\t`. By default, this value is set to `,`. +- **Double Quote**: This option determines whether two quotes in a quoted CSV value denote a single quote in the data. Set to True by default. +- **Encoding**: Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings). By default, this is set to `utf8`. +- **Escape Character**: An escape character can be used to prefix a reserved character and ensure correct parsing. A commonly used character is the backslash (`\`). For example, given the following data: + +``` +Product,Description,Price +Jeans,"Navy Blue, Bootcut, 34\"",49.99 +``` + +The backslash (`\`) is used directly before the second double quote (`"`) to indicate that it is _not_ the closing quote for the field, but rather a literal double quote character that should be included in the value (in this example, denoting the size of the jeans in inches: `34"` ). + +Leaving this field blank (default option) will disallow escaping. + +- **False Values**: A set of case-sensitive strings that should be interpreted as false values. +- **Null Values**: A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field. +- **Quote Character**: In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can handle this by wrapping a value in defined quote characters so that on read it can parse it correctly. By default, this is set to `"`. +- **Skip Rows After Header**: The number of rows to skip after the header row. +- **Skip Rows Before Header**: The number of rows to skip before the header row. +- **Strings Can Be Null**: Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself. +- **True Values**: A set of case-sensitive strings that should be interpreted as true values. ## Changelog diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 6fdf34bba10858..1929358389fb6a 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -193,6 +193,7 @@ Your token should have at least the `repo` scope. Depending on which streams you | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.5.6 | 2024-01-26 | [34503](https://github.com/airbytehq/airbyte/pull/34503) | Fix MultipleToken rotation logic | | 1.5.5 | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams | | 1.5.4 | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code | | 1.5.3 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index 1e579b009d0b34..feedde9ef93c31 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -124,7 +124,8 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| -| 1.0.0 | 2024-01-01 | [33682](https://github.com/airbytehq/airbyte/pull/33682) | Save state for stream `Board Issues` per `board` | +| 1.0.1 | 2024-01-24 | [34470](https://github.com/airbytehq/airbyte/pull/34470) | Add state checkpoint interval for all streams | +| 1.0.0 | 2024-01-01 | [33715](https://github.com/airbytehq/airbyte/pull/33715) | Save state for stream `Board Issues` per `board` | | 0.14.1 | 2023-12-19 | [33625](https://github.com/airbytehq/airbyte/pull/33625) | Skip 404 error | | 0.14.0 | 2023-12-15 | [33532](https://github.com/airbytehq/airbyte/pull/33532) | Add lookback window | | 0.13.0 | 2023-12-12 | [33353](https://github.com/airbytehq/airbyte/pull/33353) | Fix check command to check access for all available streams | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 887a141a2bbeb2..5f10949884aa9e 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -292,6 +292,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.3.2 | 2024-01-24 | [34465](https://github.com/airbytehq/airbyte/pull/34465) | Check xmin only if user selects xmin sync mode. | | 3.3.1 | 2024-01-10 | [34119](https://github.com/airbytehq/airbyte/pull/34119) | Adopt java CDK version 0.11.5. | | 3.3.0 | 2023-12-19 | [33437](https://github.com/airbytehq/airbyte/pull/33437) | Remove LEGACY state flag | | 3.2.27 | 2023-12-18 | [33605](https://github.com/airbytehq/airbyte/pull/33605) | Advance Postgres LSN for PG 14 & below. | diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 14a6d35caef5de..557100dca09957 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -223,6 +223,7 @@ Each record is marked with `is_deleted` flag when the appropriate event happens | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 5.2.1 | 2024-01-18 | [34495](https://github.com/airbytehq/airbyte/pull/34495) | Fix deadlock issue | 5.2.0 | 2024-01-18 | [34347](https://github.com/airbytehq/airbyte/pull//34347) | Add new fields invoices and subscription streams. Upgrade the CDK for better memory usage. | | 5.1.3 | 2023-12-18 | [33306](https://github.com/airbytehq/airbyte/pull/33306/) | Adding integration tests | | 5.1.2 | 2024-01-04 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | diff --git a/docs/integrations/sources/typeform.md b/docs/integrations/sources/typeform.md index 18c31df86cd328..8819ef8c4d4a46 100644 --- a/docs/integrations/sources/typeform.md +++ b/docs/integrations/sources/typeform.md @@ -90,7 +90,8 @@ API rate limits \(2 requests per second\): [https://developer.typeform.com/get-s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------| -| 1.2.3 | 2024-01-11 | [34145](https://github.com/airbytehq/airbyte/pull/34145) | prepare for airbyte-lib | +| 1.2.4 | 2024-01-24 | [34484](https://github.com/airbytehq/airbyte/pull/34484) | Fix pagination stop condition | +| 1.2.3 | 2024-01-11 | [34145](https://github.com/airbytehq/airbyte/pull/34145) | prepare for airbyte-lib | | 1.2.2 | 2023-12-12 | [33345](https://github.com/airbytehq/airbyte/pull/33345) | Fix single use refresh token authentication | | 1.2.1 | 2023-12-04 | [32775](https://github.com/airbytehq/airbyte/pull/32775) | Add 499 status code handling | | 1.2.0 | 2023-11-29 | [32745](https://github.com/airbytehq/airbyte/pull/32745) | Add `response_type` field to `responses` schema | diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 4f73ce78594089..71029b17781a18 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -128,7 +128,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.50.44 --\ + docker run --rm -v /tmp:/config airbyte/migration:0.50.45 --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/docs/release_notes/upgrading_to_destinations_v2.md b/docs/release_notes/upgrading_to_destinations_v2.md index eee8ad098b4740..f9764cac1afa06 100644 --- a/docs/release_notes/upgrading_to_destinations_v2.md +++ b/docs/release_notes/upgrading_to_destinations_v2.md @@ -158,7 +158,7 @@ For each destination connector, Destinations V2 is effective as of the following | --------------------- | --------------------- | -------------------------- | ------------------------ | | BigQuery | 1.10.2 | 2.0.6+ | November 7, 2023 | | Snowflake | 2.1.7 | 3.1.0+ | November 7, 2023 | -| Redshift | 0.6.11 | [coming soon] 2.0.0+ | [coming soon] early 2024 | +| Redshift | 0.8.0 | 2.0.0+ | March 15, 2024 | | Postgres | 0.4.0 | [coming soon] 2.0.0+ | [coming soon] early 2024 | | MySQL | 0.2.0 | [coming soon] 2.0.0+ | [coming soon] early 2024 | diff --git a/docs/understanding-airbyte/high-level-view.md b/docs/understanding-airbyte/high-level-view.md index ce45613163db9f..19cb5291da767f 100644 --- a/docs/understanding-airbyte/high-level-view.md +++ b/docs/understanding-airbyte/high-level-view.md @@ -11,16 +11,35 @@ The platform provides all the horizontal services required to configure and run Connectors are independent modules which push/pull data to/from sources and destinations. Connectors are built in accordance with the [Airbyte Specification](./airbyte-protocol.md), which describes the interface with which data can be moved between a source and a destination using Airbyte. Connectors are packaged as Docker images, which allows total flexibility over the technologies used to implement them. A more concrete diagram can be seen below: - -![3.048-Kilometer view](../.gitbook/assets/understanding_airbyte_high_level_architecture.png) - -* `UI`: An easy-to-use graphical interface for interacting with the Airbyte API. -* `WebApp Server`: Handles connection between UI and API. -* `Config Store`: Stores all the connections information \(credentials, frequency...\). -* `Scheduler Store`: Stores statuses and job information for the scheduler bookkeeping. -* `Config API`: Airbyte's main control plane. All operations in Airbyte such as creating sources, destinations, connections, managing configurations, etc.. are configured and invoked from the API. -* `Scheduler`: The scheduler takes work requests from the API and sends them to the Temporal service to parallelize. It is responsible for tracking success/failure and for triggering syncs based on the configured frequency. -* `Temporal Service`: Manages the task queue and workflows for the Scheduler. -* `Worker`: The worker connects to a source connector, pulls the data and writes it to a destination. -* `Temporary Storage`: A storage that workers can use whenever they need to spill data on a disk. - +```mermaid +--- +title: Architecture Overview +config: + theme: neutral +--- +flowchart LR + W[fa:fa-display WebApp/UI] + S[fa:fa-server Server/Config API] + D[(fa:fa-table Config & Jobs)] + T(fa:fa-calendar Temporal) + W2[1..n Airbyte Workers] + W -->|sends API requests| S + S -->|store data| D + S -->|create workflow| T + T -->|launch task| W2 + W2 -->|return job| T + W2 -->|launches| Source + W2 -->|launches| Destination +``` + +* **Web App/UI** [`airbyte-webapp`, `airbyte-proxy`]: An easy-to-use graphical interface for interacting with the Airbyte API. +* **Server/Config API** [`airbyte-server`, `airbyte-server-api`]: Handles connection between UI and API. Airbyte's main control plane. All operations in Airbyte such as creating sources, destinations, connections, managing configurations, etc.. are configured and invoked from the API. +* **Database Config & Jobs** [`airbyte-db`]: Stores all the connections information \(credentials, frequency...\). +* **Temporal Service** [`airbyte-temporal`]: Manages the task queue and workflows. +* **Worker** [`airbyte-worker`]: The worker connects to a source connector, pulls the data and writes it to a destination. + +The diagram shows the steady-state operation of Airbyte, there are components not described you'll see in your deployment: +* **Cron** [`airbyte-cron`]: Clean the server and sync logs (when using local logs) +* **Bootloader** [`airbyte-bootloader`]: Upgrade and Migrate the Database tables and confirm the enviroment is ready to work. + +This is a holistic high-level description of each component. For Airbyte deployed in Kubernetes the structure is very similar with a few changes. diff --git a/docs/using-airbyte/core-concepts/readme.md b/docs/using-airbyte/core-concepts/readme.md index dab232b3bb847d..09398b25a6183f 100644 --- a/docs/using-airbyte/core-concepts/readme.md +++ b/docs/using-airbyte/core-concepts/readme.md @@ -24,12 +24,12 @@ An Airbyte component which pulls data from a source or pushes data to a destinat A connection is an automated data pipeline that replicates data from a source to a destination. Setting up a connection enables configuration of the following parameters: -| Concept | Description | -|---------------------|---------------------------------------------------------------------------------------------------------------------| -| [Replication Frequency](/using-airbyte/core-concepts/sync-schedules.md) | When should a data sync be triggered? | -| [Destination Namespace and Stream Prefix](/using-airbyte/core-concepts/namespaces.md) | Where should the replicated data be written? | -| [Sync Mode](/using-airbyte/core-concepts/sync-modes/README.md) | How should the streams be replicated (read and written)? | -| [Schema Propagation](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How should Airbyte handle schema drift in sources? | +| Concept | Description | +|-----------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| [Replication Frequency](/using-airbyte/core-concepts/sync-schedules.md) | When should a data sync be triggered? | +| [Destination Namespace and Stream Prefix](/using-airbyte/core-concepts/namespaces.md) | Where should the replicated data be written? | +| [Sync Mode](/using-airbyte/core-concepts/sync-modes/README.md) | How should the streams be replicated (read and written)? | +| [Schema Propagation](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How should Airbyte handle schema drift in sources? | | [Catalog Selection](/cloud/managing-airbyte-cloud/configuring-connections.md#modify-streams-in-your-connection) | What data should be replicated from the source to the destination? | ## Stream @@ -83,7 +83,7 @@ Typing and deduping ensures the data emitted from sources is written into the co Typing and Deduping is the default method of transforming datasets within data warehouse and database destinations after they've been replicated. We are retaining documentation about normalization to support legacy destinations. ::: -For more details, see our [Typing & Deduping documentation](/understanding-airbyte/typing-deduping). +For more details, see our [Typing & Deduping documentation](/using-airbyte/core-concepts/typing-deduping). ## Basic Normalization diff --git a/docs/using-airbyte/workspaces.md b/docs/using-airbyte/workspaces.md index a4dfd0d524cb9f..7b211c0a0cb0c2 100644 --- a/docs/using-airbyte/workspaces.md +++ b/docs/using-airbyte/workspaces.md @@ -16,7 +16,7 @@ To add a user to your workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **Access Management**. +2. Click **Workspace** > **Access Management**. 3. Click **+ New user**. @@ -34,7 +34,7 @@ To remove a user from your workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **Access Management**. +2. Click **Workspace** > **Access Management**. 3. Click **Remove** next to the user’s email. @@ -46,7 +46,7 @@ To rename a workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **General Settings**. +2. Click **Workspace** > **General**. 3. In the **Workspace name** field, enter the new name for your workspace. @@ -58,7 +58,7 @@ To delete a workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **General Settings**. +2. Click **Workspace** > **General**. 3. In the **Delete your workspace** section, click **Delete**. diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index b73795229e37a6..f17159539f6086 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -381,18 +381,18 @@ const understandingAirbyte = { type: "category", label: "Understand Airbyte", items: [ - "understanding-airbyte/beginners-guide-to-catalog", + "understanding-airbyte/high-level-view", "understanding-airbyte/airbyte-protocol", "understanding-airbyte/airbyte-protocol-docker", - "understanding-airbyte/operations", - "understanding-airbyte/high-level-view", "understanding-airbyte/jobs", - "understanding-airbyte/tech-stack", - "understanding-airbyte/cdc", + "understanding-airbyte/database-data-catalog", + "understanding-airbyte/beginners-guide-to-catalog", "understanding-airbyte/supported-data-types", + "understanding-airbyte/operations", + "understanding-airbyte/cdc", "understanding-airbyte/json-avro-conversion", - "understanding-airbyte/database-data-catalog", "understanding-airbyte/schemaless-sources-and-destinations", + "understanding-airbyte/tech-stack", ], }; diff --git a/gradle.properties b/gradle.properties index 9d7fb27ae04cb3..b35cd4dfb0c8ca 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION=0.50.44 +VERSION=0.50.45 # NOTE: some of these values are overwritten in CI! # NOTE: if you want to override this for your local machine, set overrides in ~/.gradle/gradle.properties diff --git a/run-ab-platform.sh b/run-ab-platform.sh index 2f347c8545d508..0d01d26dda4aa7 100755 --- a/run-ab-platform.sh +++ b/run-ab-platform.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=0.50.44 +VERSION=0.50.45 # Run away from anything even a little scary set -o nounset # -u exit if a variable is not set set -o errexit # -f exit for any command failure"