diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c70e5f242943d..9522054706047 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.35.9-alpha +current_version = 0.35.13-alpha commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index 3f72c73c31e1c..1efdde598bb29 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.35.9-alpha +VERSION=0.35.13-alpha # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/.env.dev b/.env.dev index e7a4f02b7d5b0..2a6dc7eb8129b 100644 --- a/.env.dev +++ b/.env.dev @@ -27,4 +27,3 @@ SYNC_JOB_MAX_TIMEOUT_DAYS=3 # Sentry SENTRY_DSN="" - diff --git a/.github/workflows/platform-project-automation.yml b/.github/workflows/platform-project-automation.yml new file mode 100644 index 0000000000000..d53f2a508e241 --- /dev/null +++ b/.github/workflows/platform-project-automation.yml @@ -0,0 +1,34 @@ +# See https://github.com/marketplace/actions/project-beta-automations for guidance + +name: Platform Project Automation +on: + issues: + types: [labeled] + +env: + GH_PROJECT_TOKEN: ${{ secrets.PARKER_PAT_FOR_PLATFORM_PROJECT_AUTOMATION }} + ORG: airbytehq + PROJECT_ID: 6 # https://github.com/orgs/airbytehq/projects/6/views/8 + FIELD_STATUS: Status + STATUS_TODO: Todo + FIELD_DATE_ADDED: Date Added + +jobs: + add-area-platform-issues-to-platform-project: + runs-on: ubuntu-latest + name: Add area/platform issue to Platform Project + steps: + - name: Set current date env var + id: set_date + run: echo ::set-output name=CURRENT_DATE::$(date +'%Y-%m-%dT%H:%M:%S%z') + + - name: Add issue to project if labelled with area/platform + uses: leonsteinhaeuser/project-beta-automations@v1.1.0 + if: contains(github.event.issue.labels.*.name, 'area/platform') + with: + gh_token: ${{ env.GH_PROJECT_TOKEN }} + organization: ${{ env.ORG }} + project_id: ${{ env.PROJECT_ID }} + resource_node_id: ${{ github.event.issue.node_id }} + operation_mode: custom_field + custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]' diff --git a/.github/workflows/publish-external-command.yml b/.github/workflows/publish-external-command.yml new file mode 100644 index 0000000000000..2ed44926a7182 --- /dev/null +++ b/.github/workflows/publish-external-command.yml @@ -0,0 +1,113 @@ +name: Publish External Connector Image +on: + workflow_dispatch: + inputs: + connector: + description: "Airbyte Connector image" + required: true + version: + description: "Airbyte Connector version" + required: true + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + +jobs: + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + start-publish-image-runner: + name: Start Build EC2 Runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + # 80 gb disk + ec2-image-id: ami-0d648081937c75a73 + publish-image: + needs: start-publish-image-runner + runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + environment: more-secrets + steps: + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository + - run: | + echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD} + ./tools/integrations/manage.sh publish_external ${{ github.event.inputs.connector }} ${{ github.event.inputs.version }} + name: publish ${{ github.event.inputs.connector }} + id: publish + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + - name: Add Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Add Failure Comment + if: github.event.inputs.comment-id && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Slack Notification - Failure + if: failure() + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildozer + SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg + SLACK_COLOR: DC143C + SLACK_TITLE: "Failed to publish connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}" + SLACK_FOOTER: "" + # In case of self-hosted EC2 errors, remove this block. + stop-publish-image-runner: + name: Stop Build EC2 Runner + needs: + - start-publish-image-runner # required to get output from the start-runner job + - publish-image # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-publish-image-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 601663d21d9f7..7a11ce6a5de3b 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -19,6 +19,7 @@ jobs: test test-performance publish + publish-external publish-cdk gke-kube-test run-specific-test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e62b52145420f..90ac665c2b9dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,11 +11,17 @@ repos: rev: 21.11b1 hooks: - id: black + args: ["--line-length=140"] - repo: https://github.com/timothycrosley/isort rev: 5.10.1 hooks: - id: isort - args: ["--dont-follow-links", "--jobs=-1"] + args: + [ + "--settings-path=tools/python/.isort.cfg", + "--dont-follow-links", + "--jobs=-1", + ] additional_dependencies: ["colorama"] - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.5.0 @@ -34,12 +40,14 @@ repos: rev: v0.0.1a2.post1 hooks: - id: pyproject-flake8 + args: ["--config=tools/python/.flake8"] additional_dependencies: ["mccabe"] alias: flake8 - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910-1 hooks: - id: mypy + args: ["--config-file=tools/python/.mypy.ini"] exclude: | (?x)^.*( octavia-cli/unit_tests/| diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index b483fabd537a7..c2487a3b5c7ed 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -18,6 +18,7 @@ info: * All backwards incompatible changes will happen in major version bumps. We will not make backwards incompatible changes in minor version bumps. Examples of non-breaking changes (includes but not limited to...): * Adding fields to request or response bodies. * Adding new HTTP endpoints. + * All `web_backend` APIs are not considered public APIs and are not guaranteeing backwards compatibility. version: "1.0.0" title: Airbyte Configuration API @@ -53,7 +54,9 @@ tags: - name: db_migration description: Database migration related resources. - name: web_backend - description: Connection between sources and destinations. + description: | + Endpoints for the Airbyte web application. Those APIs should not be called outside the web application implementation and are not + guaranteeing any backwards compatibility. - name: health description: Healthchecks - name: deployment @@ -182,6 +185,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/workspaces/update_name: + post: + tags: + - workspace + summary: Update workspace name + operationId: updateWorkspaceName + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceUpdateName" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceRead" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/workspaces/tag_feedback_status_as_done: post: tags: @@ -1969,6 +1995,16 @@ components: type: boolean feedbackDone: type: boolean + WorkspaceUpdateName: + type: object + required: + - workspaceId + - name + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + name: + type: string WorkspaceUpdate: type: object required: diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index b573edc990369..d3966ac2a53f7 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -5,6 +5,6 @@ ENV APPLICATION airbyte-bootloader WORKDIR /app -ADD bin/${APPLICATION}-0.35.9-alpha.tar /app +ADD bin/${APPLICATION}-0.35.13-alpha.tar /app -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.13-alpha/bin/${APPLICATION}"] diff --git a/airbyte-bootloader/build.gradle b/airbyte-bootloader/build.gradle index 0707e5aa78fec..13b1a08c6f615 100644 --- a/airbyte-bootloader/build.gradle +++ b/airbyte-bootloader/build.gradle @@ -11,7 +11,9 @@ dependencies { implementation project(':airbyte-db:lib') implementation project(":airbyte-json-validation") implementation project(':airbyte-scheduler:persistence') + implementation project(':airbyte-scheduler:models') + implementation 'io.temporal:temporal-sdk:1.6.0' implementation "org.flywaydb:flyway-core:7.14.0" testImplementation "org.testcontainers:postgresql:1.15.3" diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java index 260aeab9dcdb1..ab7e0f5ec4338 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java @@ -5,6 +5,8 @@ package io.airbyte.bootloader; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; @@ -19,9 +21,14 @@ import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.jobs.JobsDatabaseInstance; import io.airbyte.db.instance.jobs.JobsDatabaseMigrator; +import io.airbyte.scheduler.models.Job; +import io.airbyte.scheduler.models.JobStatus; import io.airbyte.scheduler.persistence.DefaultJobPersistence; import io.airbyte.scheduler.persistence.JobPersistence; import io.airbyte.validation.json.JsonValidationException; +import io.temporal.client.WorkflowClient; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; import java.io.IOException; import java.util.Optional; import java.util.UUID; @@ -47,10 +54,12 @@ public class BootloaderApp { private final Configs configs; private Runnable postLoadExecution; + private FeatureFlags featureFlags; @VisibleForTesting - public BootloaderApp(Configs configs) { + public BootloaderApp(Configs configs, FeatureFlags featureFlags) { this.configs = configs; + this.featureFlags = featureFlags; } /** @@ -61,9 +70,10 @@ public BootloaderApp(Configs configs) { * @param configs * @param postLoadExecution */ - public BootloaderApp(Configs configs, Runnable postLoadExecution) { + public BootloaderApp(Configs configs, Runnable postLoadExecution, FeatureFlags featureFlags) { this.configs = configs; this.postLoadExecution = postLoadExecution; + this.featureFlags = featureFlags; } public BootloaderApp() { @@ -80,6 +90,7 @@ public BootloaderApp() { e.printStackTrace(); } }; + featureFlags = new EnvVariableFeatureFlags(); } public void load() throws Exception { @@ -207,4 +218,16 @@ private static void runFlywayMigration(final Configs configs, final Database con } } + private static void cleanupZombies(final JobPersistence jobPersistence) throws IOException { + final Configs configs = new EnvConfigs(); + WorkflowClient wfClient = + WorkflowClient.newInstance(WorkflowServiceStubs.newInstance( + WorkflowServiceStubsOptions.newBuilder().setTarget(configs.getTemporalHost()).build())); + for (final Job zombieJob : jobPersistence.listJobsWithStatus(JobStatus.RUNNING)) { + LOGGER.info("Kill zombie job {} for connection {}", zombieJob.getId(), zombieJob.getScope()); + wfClient.newUntypedWorkflowStub("sync_" + zombieJob.getId()) + .terminate("Zombie"); + } + } + } diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java index 18ccdf2fc1ff4..c062764e89d9c 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; @@ -53,6 +54,9 @@ void testBootloaderAppBlankDb() throws Exception { when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); + val mockedFeatureFlags = mock(FeatureFlags.class); + when(mockedFeatureFlags.usesNewScheduler()).thenReturn(false); + // Although we are able to inject mocked configs into the Bootloader, a particular migration in the // configs database // requires the env var to be set. Flyway prevents injection, so we dynamically set this instead. @@ -60,7 +64,7 @@ void testBootloaderAppBlankDb() throws Exception { environmentVariables.set("DATABASE_PASSWORD", "docker"); environmentVariables.set("DATABASE_URL", container.getJdbcUrl()); - val bootloader = new BootloaderApp(mockedConfigs); + val bootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags); bootloader.load(); val jobDatabase = new JobsDatabaseInstance( @@ -127,7 +131,10 @@ void testPostLoadExecutionExecutes() throws Exception { when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); - new BootloaderApp(mockedConfigs, () -> testTriggered.set(true)).load(); + val mockedFeatureFlags = mock(FeatureFlags.class); + when(mockedFeatureFlags.usesNewScheduler()).thenReturn(false); + + new BootloaderApp(mockedConfigs, () -> testTriggered.set(true), mockedFeatureFlags).load(); assertTrue(testTriggered.get()); } diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java index a0637930edd8f..0376a1b9e18ac 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java @@ -119,6 +119,12 @@ public List listConfigs(final AirbyteConfig configType, final Class cl return configs.values().stream().map(json -> Jsons.object(json, clazz)).collect(Collectors.toList()); } + @Override + public ConfigWithMetadata getConfigWithMetadata(final AirbyteConfig configType, final String configId, final Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + throw new UnsupportedOperationException("Yaml Seed Config doesn't support metadata"); + } + @Override public List> listConfigsWithMetadata(final AirbyteConfig configType, final Class clazz) throws JsonValidationException, IOException { diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json index 95b7cf1790a86..82ccc53128a92 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "356668e2-7e34-47f3-a3b0-67a8a481b692", "name": "Google PubSub", "dockerRepository": "airbyte/destination-pubsub", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pubsub", "icon": "googlepubsub.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json index 7c675af47a73d..6c54519659eb8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72", "name": "Oracle (Alpha)", "dockerRepository": "airbyte/destination-oracle", - "dockerImageTag": "0.1.11", + "dockerImageTag": "0.1.13", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", "icon": "oracle.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json index bc563225389e8..427e888c57795 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", "name": "Snowflake", "dockerRepository": "airbyte/destination-snowflake", - "dockerImageTag": "0.4.4", + "dockerImageTag": "0.4.6", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", "icon": "snowflake.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json index 960737ad74792..1cf568381ea74 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", "name": "Looker", "dockerRepository": "airbyte/source-looker", - "dockerImageTag": "0.2.6", + "dockerImageTag": "0.2.7", "documentationUrl": "https://docs.airbyte.io/integrations/sources/looker", "icon": "looker.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json index 6b470bcf188cb..624eefa1d0ae5 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", "name": "HubSpot", "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.35", + "dockerImageTag": "0.1.37", "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", "icon": "hubspot.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59c5501b-9f95-411e-9269-7143c939adbd.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59c5501b-9f95-411e-9269-7143c939adbd.json index 1a27858cc21d8..21c0faacca2bf 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59c5501b-9f95-411e-9269-7143c939adbd.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59c5501b-9f95-411e-9269-7143c939adbd.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "59c5501b-9f95-411e-9269-7143c939adbd", "name": "Bigcommerce", "dockerRepository": "airbyte/source-bigcommerce", - "dockerImageTag": "0.1.2", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigcommerce", "icon": "bigcommerce.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json index d56bf69b98ace..9058351ef75b9 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", "name": "Microsoft SQL Server (MSSQL)", "dockerRepository": "airbyte/source-mssql", - "dockerImageTag": "0.3.13", + "dockerImageTag": "0.3.14", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mssql", "icon": "mssql.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json index 0b6d16bd6ed1f..59270745453bf 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "c2281cee-86f9-4a86-bb48-d23286b4c7bd", "name": "Slack", "dockerRepository": "airbyte/source-slack", - "dockerImageTag": "0.1.13", + "dockerImageTag": "0.1.14", "documentationUrl": "https://docs.airbyte.io/integrations/sources/slack", "icon": "slack.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json index a85e89d23cdee..bab92cf626815 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", "name": "Postgres", "dockerRepository": "airbyte/source-postgres", - "dockerImageTag": "0.4.2", + "dockerImageTag": "0.4.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres", "icon": "postgresql.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json index 2fccee5cd4e1c..392053888a1b7 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2", "name": "Snowflake", "dockerRepository": "airbyte/source-snowflake", - "dockerImageTag": "0.1.5", + "dockerImageTag": "0.1.6", "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake", "icon": "snowflake.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json index 845729a16af23..aba0262e78883 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6", "name": "Redshift", "dockerRepository": "airbyte/source-redshift", - "dockerImageTag": "0.3.6", + "dockerImageTag": "0.3.7", "documentationUrl": "https://docs.airbyte.io/integrations/sources/redshift", "icon": "redshift.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json index 346029664e420..5e89c7638aae5 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", "name": "GitHub", "dockerRepository": "airbyte/source-github", - "dockerImageTag": "0.2.8", + "dockerImageTag": "0.2.15", "documentationUrl": "https://docs.airbyte.io/integrations/sources/github", "icon": "github.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index dc4f0cf342adf..7bd1fb3b4888e 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -66,7 +66,7 @@ - name: Google PubSub destinationDefinitionId: 356668e2-7e34-47f3-a3b0-67a8a481b692 dockerRepository: airbyte/destination-pubsub - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/pubsub icon: googlepubsub.svg - name: Kafka @@ -126,7 +126,7 @@ - name: Oracle destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72 dockerRepository: airbyte/destination-oracle - dockerImageTag: 0.1.12 + dockerImageTag: 0.1.13 documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle icon: oracle.svg - name: Postgres @@ -179,7 +179,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.4 + dockerImageTag: 0.4.6 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg - name: MariaDB ColumnStore diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index f26e3e7e9fd33..ebd87e918edb6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1508,7 +1508,7 @@ - "overwrite" - "append" $schema: "http://json-schema.org/draft-07/schema#" -- dockerImage: "airbyte/destination-pubsub:0.1.1" +- dockerImage: "airbyte/destination-pubsub:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/pubsub" connectionSpecification: @@ -1523,11 +1523,11 @@ properties: project_id: type: "string" - description: "The GCP project ID for the project containing the target PubSub" + description: "The GCP project ID for the project containing the target PubSub." title: "Project ID" topic_id: type: "string" - description: "PubSub topic ID in the given GCP project ID" + description: "The PubSub topic ID in the given GCP project ID." title: "PubSub Topic ID" credentials_json: type: "string" @@ -2591,7 +2591,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-oracle:0.1.12" +- dockerImage: "airbyte/destination-oracle:0.1.13" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/oracle" connectionSpecification: @@ -2607,12 +2607,12 @@ properties: host: title: "Host" - description: "Hostname of the database." + description: "The hostname of the database." type: "string" order: 0 port: title: "Port" - description: "Port of the database." + description: "The port of the database." type: "integer" minimum: 0 maximum: 65536 @@ -2622,28 +2622,29 @@ order: 1 sid: title: "SID" - description: "SID" + description: "The System Identifier uniquely distinguishes the instance\ + \ from any other instance on the same computer." type: "string" order: 2 username: title: "User" - description: "Username to use to access the database. This user must have\ - \ CREATE USER privileges in the database." + description: "The username to access the database. This user must have CREATE\ + \ USER privileges in the database." type: "string" order: 3 password: title: "Password" - description: "Password associated with the username." + description: "The password associated with the username." type: "string" airbyte_secret: true order: 4 schema: title: "Default Schema" - description: "The default schema tables are written to if the source does\ - \ not specify a namespace. The usual value for this field is \"airbyte\"\ - . In Oracle, schemas and users are the same thing, so the \"user\" parameter\ - \ is used as the login credentials and this is used for the default Airbyte\ - \ message schema." + description: "The default schema is used as the target schema for all statements\ + \ issued from the connection that do not explicitly specify a schema name.\ + \ The usual value for this field is \"airbyte\". In Oracle, schemas and\ + \ users are the same thing, so the \"user\" parameter is used as the login\ + \ credentials and this is used for the default Airbyte message schema." type: "string" examples: - "airbyte" @@ -2652,7 +2653,8 @@ encryption: title: "Encryption" type: "object" - description: "Encryption method to use when communicating with the database" + description: "The encryption method which is used when communicating with\ + \ the database." order: 6 oneOf: - title: "Unencrypted" @@ -2667,9 +2669,9 @@ enum: - "unencrypted" default: "unencrypted" - - title: "Native Network Ecryption (NNE)" + - title: "Native Network Encryption (NNE)" additionalProperties: false - description: "Native network encryption gives you the ability to encrypt\ + description: "The native network encryption gives you the ability to encrypt\ \ database connections, without the configuration overhead of TCP/IP\ \ and SSL/TLS and without the need to open and listen on different ports." required: @@ -2683,8 +2685,7 @@ default: "client_nne" encryption_algorithm: type: "string" - description: "This parameter defines the encryption algorithm to be\ - \ used" + description: "This parameter defines the database encryption algorithm." title: "Encryption Algorithm" default: "AES256" enum: @@ -2693,7 +2694,7 @@ - "3DES168" - title: "TLS Encrypted (verify certificate)" additionalProperties: false - description: "Verify and use the cert provided by the server." + description: "Verify and use the certificate provided by the server." required: - "encryption_method" - "ssl_certificate" @@ -2707,7 +2708,7 @@ ssl_certificate: title: "SSL PEM file" description: "Privacy Enhanced Mail (PEM) files are concatenated certificate\ - \ containers frequently used in certificate installations" + \ containers frequently used in certificate installations." type: "string" airbyte_secret: true multiline: true @@ -3786,7 +3787,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.4.4" +- dockerImage: "airbyte/destination-snowflake:0.4.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" connectionSpecification: @@ -3804,8 +3805,8 @@ additionalProperties: true properties: host: - description: "Host domain of the snowflake instance (must include the account,\ - \ region, cloud environment, and end with snowflakecomputing.com)." + description: "The host domain of the snowflake instance (must include the\ + \ account, region, cloud environment, and end with snowflakecomputing.com)." examples: - "accountname.us-east-2.aws.snowflakecomputing.com" type: "string" @@ -3833,10 +3834,10 @@ title: "Database" order: 3 schema: - description: "The default Snowflake schema tables are written to if the\ - \ source does not specify a namespace. Schema name would be transformed\ - \ to allowed by Snowflake if it not follow Snowflake Naming Conventions\ - \ https://docs.airbyte.io/integrations/destinations/snowflake#notes-about-snowflake-naming-conventions " + description: "The default schema is used as the target schema for all statements\ + \ issued from the connection that do not explicitly specify a schema name..\ + \ Schema name would be transformed to allowed by Snowflake if it not follow\ + \ Snowflake Naming Conventions https://docs.airbyte.io/integrations/destinations/snowflake#notes-about-snowflake-naming-conventions " examples: - "AIRBYTE_SCHEMA" type: "string" @@ -3850,16 +3851,23 @@ title: "Username" order: 5 password: - description: "Password associated with the username." + description: "The password associated with the username." type: "string" airbyte_secret: true title: "Password" order: 6 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + title: "JDBC URL Params" + type: "string" + order: 7 loading_method: type: "object" title: "Loading Method" - description: "Loading method used to send data to Snowflake." - order: 7 + description: "The loading method used to send data to Snowflake." + order: 8 oneOf: - title: "[Recommended] Internal Staging" additionalProperties: false @@ -3917,8 +3925,8 @@ title: "S3 Bucket Region" type: "string" default: "" - description: "The region of the S3 staging bucket to use if utilising\ - \ a copy strategy." + description: "The region of the S3 staging bucket which is used when\ + \ utilising a copy strategy." enum: - "" - "us-east-1" diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 01f7e4b661b4d..15e93f2b31a48 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -77,7 +77,7 @@ - name: BigCommerce sourceDefinitionId: 59c5501b-9f95-411e-9269-7143c939adbd dockerRepository: airbyte/source-bigcommerce - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/bigcommerce icon: bigcommerce.svg sourceType: api @@ -154,7 +154,7 @@ - name: Delighted sourceDefinitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635 dockerRepository: airbyte/source-delighted - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/delighted icon: delighted.svg sourceType: api @@ -231,7 +231,7 @@ - name: GitHub sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e dockerRepository: airbyte/source-github - dockerImageTag: 0.2.14 + dockerImageTag: 0.2.15 documentationUrl: https://docs.airbyte.io/integrations/sources/github icon: github.svg sourceType: api @@ -252,7 +252,7 @@ - name: Google Analytics sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 dockerRepository: airbyte/source-google-analytics-v4 - dockerImageTag: 0.1.15 + dockerImageTag: 0.1.16 documentationUrl: https://docs.airbyte.io/integrations/sources/google-analytics-v4 icon: google-analytics.svg sourceType: api @@ -307,7 +307,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.35 + dockerImageTag: 0.1.37 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api @@ -390,7 +390,7 @@ - name: Looker sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c dockerRepository: airbyte/source-looker - dockerImageTag: 0.2.6 + dockerImageTag: 0.2.7 documentationUrl: https://docs.airbyte.io/integrations/sources/looker icon: looker.svg sourceType: api @@ -418,7 +418,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.3.13 + dockerImageTag: 0.3.14 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -564,7 +564,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 0.4.2 + dockerImageTag: 0.4.4 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -606,7 +606,7 @@ - name: Redshift sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 dockerRepository: airbyte/source-redshift - dockerImageTag: 0.3.6 + dockerImageTag: 0.3.7 documentationUrl: https://docs.airbyte.io/integrations/sources/redshift icon: redshift.svg sourceType: database @@ -634,7 +634,7 @@ - name: Salesforce sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 dockerRepository: airbyte/source-salesforce - dockerImageTag: 0.1.20 + dockerImageTag: 0.1.21 documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce icon: salesforce.svg sourceType: api @@ -669,7 +669,7 @@ - name: Slack sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd dockerRepository: airbyte/source-slack - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.io/integrations/sources/slack icon: slack.svg sourceType: api @@ -690,7 +690,7 @@ - name: Snowflake sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 dockerRepository: airbyte/source-snowflake - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake icon: snowflake.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 4222a70dbd4d8..fe2e26ec09171 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -665,7 +665,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-bigcommerce:0.1.3" +- dockerImage: "airbyte/source-bigcommerce:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bigcommerce" connectionSpecification: @@ -1307,7 +1307,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-delighted:0.1.2" +- dockerImage: "airbyte/source-delighted:0.1.3" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -2160,7 +2160,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-github:0.2.14" +- dockerImage: "airbyte/source-github:0.2.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/github" connectionSpecification: @@ -2449,7 +2449,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-analytics-v4:0.1.15" +- dockerImage: "airbyte/source-google-analytics-v4:0.1.16" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/google-analytics-v4" connectionSpecification: @@ -2476,6 +2476,20 @@ \ will not be replicated." examples: - "2020-06-01" + window_in_days: + type: "integer" + title: "Window in days" + description: "The amount of days for each data-chunk beginning from start_date.\ + \ Bigger the value - faster the fetch. (Min=1, as for a Day; Max=364,\ + \ as for a Year)." + examples: + - 30 + - 60 + - 90 + - 120 + - 200 + - 364 + default: 1 custom_reports: order: 3 type: "string" @@ -2487,6 +2501,7 @@ order: 0 type: "object" title: "Credentials" + description: "Credentials for the service" oneOf: - title: "Authenticate via Google (Oauth)" type: "object" @@ -3067,7 +3082,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.1.35" +- dockerImage: "airbyte/source-hubspot:0.1.37" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: @@ -3988,7 +4003,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-looker:0.2.6" +- dockerImage: "airbyte/source-looker:0.2.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/looker" connectionSpecification: @@ -4217,7 +4232,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.3.13" +- dockerImage: "airbyte/source-mssql:0.3.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -4963,7 +4978,7 @@ airbyte_secret: true order: 4 jdbc_url_params: - description: "Additional properties to pass to the jdbc url string when\ + description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." title: "JDBC URL Params" @@ -5895,9 +5910,9 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:0.4.2" +- dockerImage: "airbyte/source-postgres:0.4.4" spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/postgres" + documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" title: "Postgres Source Spec" @@ -5982,7 +5997,7 @@ description: "Logical replication uses the Postgres write-ahead log (WAL)\ \ to detect inserts, updates, and deletes. This needs to be configured\ \ on the source database itself. Only available on Postgres 10 and above.\ - \ Read the Postgres Source docs for more information." required: - "method" @@ -5998,11 +6013,12 @@ order: 0 plugin: type: "string" + title: "Plugin" description: "A logical decoding plug-in installed on the PostgreSQL\ \ server. `pgoutput` plug-in is used by default.\nIf replication\ \ table contains a lot of big jsonb values it is recommended to\ \ use `wal2json` plug-in. For more information about `wal2json`\ - \ plug-in read Postgres Source docs." enum: - "pgoutput" @@ -6011,10 +6027,12 @@ order: 1 replication_slot: type: "string" + title: "Replication Slot" description: "A plug-in logical replication slot." order: 2 publication: type: "string" + title: "Publication" description: "A Postgres publication used for consuming changes." order: 3 tunnel_method: @@ -6315,7 +6333,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-redshift:0.3.6" +- dockerImage: "airbyte/source-redshift:0.3.7" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift" connectionSpecification: @@ -6335,6 +6353,7 @@ description: "Host Endpoint of the Redshift Cluster (must include the cluster-id,\ \ region and end with .redshift.amazonaws.com)." type: "string" + order: 1 port: title: "Port" description: "Port of the database." @@ -6344,21 +6363,37 @@ default: 5439 examples: - "5439" + order: 2 database: title: "Database" description: "Name of the database." type: "string" examples: - "master" + order: 3 + schemas: + title: "Schemas" + description: "The list of schemas to sync from. Specify one or more explicitly\ + \ or keep empty to process all schemas. Schema names are case sensitive." + type: "array" + items: + type: "string" + minItems: 0 + uniqueItems: true + examples: + - "public" + order: 4 username: title: "Username" description: "Username to use to access the database." type: "string" + order: 5 password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true + order: 6 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] @@ -6731,7 +6766,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:0.1.20" +- dockerImage: "airbyte/source-salesforce:0.1.21" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce" connectionSpecification: @@ -7119,7 +7154,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-slack:0.1.13" +- dockerImage: "airbyte/source-slack:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/slack" connectionSpecification: @@ -7307,7 +7342,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-snowflake:0.1.5" +- dockerImage: "airbyte/source-snowflake:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake" connectionSpecification: @@ -7373,6 +7408,13 @@ airbyte_secret: true title: "Password" order: 6 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + title: "JDBC URL Params" + type: "string" + order: 7 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] diff --git a/airbyte-config/models/src/main/resources/types/AttemptFailureSummary.yaml b/airbyte-config/models/src/main/resources/types/AttemptFailureSummary.yaml new file mode 100644 index 0000000000000..a383f2cd615fa --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/AttemptFailureSummary.yaml @@ -0,0 +1,18 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/AttemptFailureSummary.yaml +title: AttemptFailureSummary +description: Attempt-level summarization of failures that occurred during a sync workflow. +type: object +additionalProperties: false +required: + - failures +properties: + failures: + description: Ordered list of failures that occurred during the attempt. + type: array + items: + "$ref": FailureReason.yaml + partialSuccess: + description: True if the number of committed records for this attempt was greater than 0. False if 0 records were committed. + type: boolean diff --git a/airbyte-config/models/src/main/resources/types/FailureReason.yaml b/airbyte-config/models/src/main/resources/types/FailureReason.yaml new file mode 100644 index 0000000000000..18ea82b465f46 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/FailureReason.yaml @@ -0,0 +1,44 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/FailureReason.yaml +title: FailureSummary +type: object +required: + - failureOrigin + - timestamp +additionalProperties: false +properties: + failureOrigin: + description: Indicates where the error originated. If not set, the origin of error is not well known. + type: string + enum: + - unknown + - source + - destination + - replicationWorker + - persistence + - normalization + - dbt + failureType: + description: Categorizes well known errors into types for programmatic handling. If not set, the type of error is not well known. + type: string + enum: + - unknown + - userError + - systemError + - transient + internalMessage: + description: Human readable failure description for consumption by technical system operators, like Airbyte engineers or OSS users. + type: string + externalMessage: + description: Human readable failure description for presentation in the UI to non-technical users. + type: string + metadata: + description: Key-value pairs of relevant data + type: object + additionalProperties: true + stacktrace: + description: Raw stacktrace associated with the failure. + type: string + timestamp: + type: integer diff --git a/airbyte-config/models/src/main/resources/types/ReplicationOutput.yaml b/airbyte-config/models/src/main/resources/types/ReplicationOutput.yaml index 76b1deb6de4fb..0182c02594056 100644 --- a/airbyte-config/models/src/main/resources/types/ReplicationOutput.yaml +++ b/airbyte-config/models/src/main/resources/types/ReplicationOutput.yaml @@ -16,3 +16,7 @@ properties: "$ref": State.yaml output_catalog: existingJavaType: io.airbyte.protocol.models.ConfiguredAirbyteCatalog + failures: + type: array + items: + "$ref": FailureReason.yaml diff --git a/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml b/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml index 727a83de0554a..79b2471b69e07 100644 --- a/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml +++ b/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml @@ -16,3 +16,7 @@ properties: "$ref": State.yaml output_catalog: existingJavaType: io.airbyte.protocol.models.ConfiguredAirbyteCatalog + failures: + type: array + items: + "$ref": FailureReason.yaml diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java index 2a9cad477d1c7..c88145db87096 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java @@ -19,6 +19,9 @@ public interface ConfigPersistence { List listConfigs(AirbyteConfig configType, Class clazz) throws JsonValidationException, IOException; + ConfigWithMetadata getConfigWithMetadata(AirbyteConfig configType, String configId, Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException; + List> listConfigsWithMetadata(AirbyteConfig configType, Class clazz) throws JsonValidationException, IOException; void writeConfig(AirbyteConfig configType, String configId, T config) throws JsonValidationException, IOException; diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java index 28baf348311b7..43b3d07c7b138 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java @@ -139,7 +139,7 @@ private StandardDestinationDefinition getStandardDestinationDefinition(final Str return result.get(0).getConfig(); } - private SourceConnection getSourceConnection(String configId) throws IOException, ConfigNotFoundException { + private SourceConnection getSourceConnection(final String configId) throws IOException, ConfigNotFoundException { final List> result = listSourceConnectionWithMetadata(Optional.of(UUID.fromString(configId))); validate(configId, result, ConfigSchema.SOURCE_CONNECTION); return result.get(0).getConfig(); @@ -188,7 +188,7 @@ private List connectionOperationIds(final UUID connectionId) throws IOExce .fetch()); final List ids = new ArrayList<>(); - for (Record record : result) { + for (final Record record : result) { ids.add(record.get(CONNECTION_OPERATION.OPERATION_ID)); } @@ -204,6 +204,14 @@ private void validate(final String configId, final List ConfigWithMetadata validateAndReturn(final String configId, + final List> result, + final AirbyteConfig airbyteConfig) + throws ConfigNotFoundException { + validate(configId, result, airbyteConfig); + return result.get(0); + } + @Override public List listConfigs(final AirbyteConfig configType, final Class clazz) throws JsonValidationException, IOException { final List config = new ArrayList<>(); @@ -211,6 +219,35 @@ public List listConfigs(final AirbyteConfig configType, final Class cl return config; } + @Override + public ConfigWithMetadata getConfigWithMetadata(final AirbyteConfig configType, final String configId, final Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + final Optional configIdOpt = Optional.of(UUID.fromString(configId)); + if (configType == ConfigSchema.STANDARD_WORKSPACE) { + return (ConfigWithMetadata) validateAndReturn(configId, listStandardWorkspaceWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + return (ConfigWithMetadata) validateAndReturn(configId, listStandardSourceDefinitionWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + return (ConfigWithMetadata) validateAndReturn(configId, listStandardDestinationDefinitionWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.SOURCE_CONNECTION) { + return (ConfigWithMetadata) validateAndReturn(configId, listSourceConnectionWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.DESTINATION_CONNECTION) { + return (ConfigWithMetadata) validateAndReturn(configId, listDestinationConnectionWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.SOURCE_OAUTH_PARAM) { + return (ConfigWithMetadata) validateAndReturn(configId, listSourceOauthParamWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.DESTINATION_OAUTH_PARAM) { + return (ConfigWithMetadata) validateAndReturn(configId, listDestinationOauthParamWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.STANDARD_SYNC_OPERATION) { + return (ConfigWithMetadata) validateAndReturn(configId, listStandardSyncOperationWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.STANDARD_SYNC) { + return (ConfigWithMetadata) validateAndReturn(configId, listStandardSyncWithMetadata(configIdOpt), configType); + } else if (configType == ConfigSchema.STANDARD_SYNC_STATE) { + return (ConfigWithMetadata) validateAndReturn(configId, listStandardSyncStateWithMetadata(configIdOpt), configType); + } else { + throw new IllegalArgumentException("Unknown Config Type " + configType); + } + } + @Override public List> listConfigsWithMetadata(final AirbyteConfig configType, final Class clazz) throws IOException { final List> configWithMetadata = new ArrayList<>(); @@ -258,7 +295,7 @@ private List> listStandardWorkspaceWithMet for (final Record record : result) { final List notificationList = new ArrayList<>(); final List fetchedNotifications = Jsons.deserialize(record.get(WORKSPACE.NOTIFICATIONS).data(), List.class); - for (Object notification : fetchedNotifications) { + for (final Object notification : fetchedNotifications) { notificationList.add(Jsons.convertValue(notification, Notification.class)); } final StandardWorkspace workspace = buildStandardWorkspace(record, notificationList); diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistence.java index 138fc18129257..9996cb071aa06 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistence.java @@ -132,6 +132,29 @@ public List listConfigs(final AirbyteConfig configType, final Class cl .collect(Collectors.toList()); } + @Override + public ConfigWithMetadata getConfigWithMetadata(final AirbyteConfig configType, final String configId, final Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId)) + .fetch()); + + if (result.isEmpty()) { + throw new ConfigNotFoundException(configType, configId); + } else if (result.size() > 1) { + throw new IllegalStateException(String.format("Multiple %s configs found for ID %s: %s", configType, configId, result)); + } + + final Record record = result.get(0); + return new ConfigWithMetadata<>( + record.get(AIRBYTE_CONFIGS.CONFIG_ID), + record.get(AIRBYTE_CONFIGS.CONFIG_TYPE), + record.get(AIRBYTE_CONFIGS.CREATED_AT).toInstant(), + record.get(AIRBYTE_CONFIGS.UPDATED_AT).toInstant(), + Jsons.deserialize(result.get(0).get(AIRBYTE_CONFIGS.CONFIG_BLOB).data(), clazz)); + } + @Override public List> listConfigsWithMetadata(final AirbyteConfig configType, final Class clazz) throws IOException { final Result results = database.query(ctx -> ctx.select(asterisk()) diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/FileSystemConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/FileSystemConfigPersistence.java index e91f335d15eb1..82395b8d13c50 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/FileSystemConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/FileSystemConfigPersistence.java @@ -79,6 +79,12 @@ public List listConfigs(final AirbyteConfig configType, final Class cl } } + @Override + public ConfigWithMetadata getConfigWithMetadata(final AirbyteConfig configType, final String configId, final Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + throw new UnsupportedOperationException("File Persistence doesn't support metadata"); + } + @Override public List> listConfigsWithMetadata(final AirbyteConfig configType, final Class clazz) throws JsonValidationException, IOException { diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ValidatingConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ValidatingConfigPersistence.java index 81ac50832e84c..5cd4dd594849b 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ValidatingConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ValidatingConfigPersistence.java @@ -48,6 +48,14 @@ public List listConfigs(final AirbyteConfig configType, final Class cl return configs; } + @Override + public ConfigWithMetadata getConfigWithMetadata(final AirbyteConfig configType, final String configId, final Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + final ConfigWithMetadata config = decoratedPersistence.getConfigWithMetadata(configType, configId, clazz); + validateJson(config.getConfig(), configType); + return config; + } + @Override public List> listConfigsWithMetadata(final AirbyteConfig configType, final Class clazz) throws JsonValidationException, IOException { diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java index f2a8042426fc9..8a3ef90a6d9dd 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java @@ -86,6 +86,21 @@ public void testWriteAndGetConfig() throws Exception { .hasSameElementsAs(List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3)); } + @Test + public void testGetConfigWithMetadata() throws Exception { + final Instant now = Instant.now().minus(Duration.ofSeconds(1)); + writeDestination(configPersistence, DESTINATION_S3); + final ConfigWithMetadata configWithMetadata = configPersistence.getConfigWithMetadata( + STANDARD_DESTINATION_DEFINITION, + DESTINATION_S3.getDestinationDefinitionId().toString(), + StandardDestinationDefinition.class); + assertEquals("STANDARD_DESTINATION_DEFINITION", configWithMetadata.getConfigType()); + assertTrue(configWithMetadata.getCreatedAt().isAfter(now)); + assertTrue(configWithMetadata.getUpdatedAt().isAfter(now)); + assertEquals(DESTINATION_S3.getDestinationDefinitionId().toString(), configWithMetadata.getConfigId()); + assertEquals(DESTINATION_S3, configWithMetadata.getConfig()); + } + @Test public void testListConfigWithMetadata() throws Exception { final Instant now = Instant.now().minus(Duration.ofSeconds(1)); diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceTest.java index 58456ec213b46..07ae8050e9cbe 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceTest.java @@ -78,6 +78,21 @@ public void testWriteAndGetConfig() throws Exception { configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); } + @Test + public void testGetConfigWithMetadata() throws Exception { + final Instant now = Instant.now().minus(Duration.ofSeconds(1)); + writeDestination(configPersistence, DESTINATION_S3); + final ConfigWithMetadata configWithMetadata = configPersistence.getConfigWithMetadata( + STANDARD_DESTINATION_DEFINITION, + DESTINATION_S3.getDestinationDefinitionId().toString(), + StandardDestinationDefinition.class); + assertEquals("STANDARD_DESTINATION_DEFINITION", configWithMetadata.getConfigType()); + assertTrue(configWithMetadata.getCreatedAt().isAfter(now)); + assertTrue(configWithMetadata.getUpdatedAt().isAfter(now)); + assertEquals(DESTINATION_S3.getDestinationDefinitionId().toString(), configWithMetadata.getConfigId()); + assertEquals(DESTINATION_S3, configWithMetadata.getConfig()); + } + @Test public void testListConfigWithMetadata() throws Exception { final Instant now = Instant.now().minus(Duration.ofSeconds(1)); diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 60022b06d8aaa..7b5650eb51692 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -26,12 +26,12 @@ RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] htt RUN apt-get update && apt-get install -y kubectl ENV APPLICATION airbyte-container-orchestrator -ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}" +ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.13-alpha/bin/${APPLICATION}" WORKDIR /app # Move orchestrator app -ADD bin/${APPLICATION}-0.35.9-alpha.tar /app +ADD bin/${APPLICATION}-0.35.13-alpha.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.13-alpha/bin/${APPLICATION}"] diff --git a/airbyte-integrations/bases/base-java/Dockerfile b/airbyte-integrations/bases/base-java/Dockerfile index b504b46476085..ea1844de4e642 100644 --- a/airbyte-integrations/bases/base-java/Dockerfile +++ b/airbyte-integrations/bases/base-java/Dockerfile @@ -13,6 +13,8 @@ ENV AIRBYTE_DISCOVER_CMD "/airbyte/javabase.sh --discover" ENV AIRBYTE_READ_CMD "/airbyte/javabase.sh --read" ENV AIRBYTE_WRITE_CMD "/airbyte/javabase.sh --write" +ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" + ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" ENTRYPOINT ["/airbyte/base.sh"] diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index 136e0f8c19d49..2ba85b21ccd79 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -7,6 +7,7 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-commons-cli') implementation project(':airbyte-json-validation') + api 'io.sentry:sentry:5.6.0' implementation 'commons-cli:commons-cli:1.4' implementation 'org.apache.sshd:sshd-mina:2.7.0' diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java index c0ff0297ee2a8..2e0aa336595d2 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java @@ -15,7 +15,11 @@ import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.validation.json.JsonSchemaValidator; +import io.sentry.ITransaction; +import io.sentry.Sentry; +import io.sentry.SpanStatus; import java.nio.file.Path; +import java.util.Map; import java.util.Optional; import java.util.Scanner; import java.util.Set; @@ -73,10 +77,31 @@ public IntegrationRunner(final Source source) { } public void run(final String[] args) throws Exception { - LOGGER.info("Running integration: {}", integration.getClass().getName()); + initSentry(); final IntegrationConfig parsed = cliParser.parse(args); + final ITransaction transaction = Sentry.startTransaction( + integration.getClass().getSimpleName(), + parsed.getCommand().toString(), + true); + try { + runInternal(transaction, parsed); + transaction.finish(SpanStatus.OK); + } catch (final Exception e) { + transaction.setThrowable(e); + transaction.finish(SpanStatus.INTERNAL_ERROR); + throw e; + } finally { + /* + * This finally block may not run, probably because the container can be terminated by the worker. + * So the transaction should always be finished in the try and catch blocks. + */ + transaction.finish(); + } + } + public void runInternal(final ITransaction transaction, final IntegrationConfig parsed) throws Exception { + LOGGER.info("Running integration: {}", integration.getClass().getName()); LOGGER.info("Command: {}", parsed.getCommand()); LOGGER.info("Integration config: {}", parsed); @@ -169,4 +194,21 @@ private static T parseConfig(final Path path, final Class klass) { return Jsons.object(jsonNode, klass); } + private static void initSentry() { + final Map env = System.getenv(); + final String connector = env.getOrDefault("APPLICATION", "unknown"); + final String version = env.getOrDefault("APPLICATION_VERSION", "unknown"); + final boolean enableSentry = Boolean.parseBoolean(env.getOrDefault("ENABLE_SENTRY", "false")); + + // https://docs.sentry.io/platforms/java/configuration/ + Sentry.init(options -> { + options.setDsn(env.getOrDefault("SENTRY_DSN", "")); + options.setEnableExternalConfiguration(true); + options.setTracesSampleRate(enableSentry ? 1.0 : 0.0); + options.setRelease(String.format("%s@%s", connector, version)); + options.setTag("connector", connector); + options.setTag("connector_version", version); + }); + } + } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java new file mode 100644 index 0000000000000..9c512e6273056 --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.sentry; + +import io.sentry.ISpan; +import io.sentry.Sentry; +import io.sentry.SpanStatus; +import java.util.concurrent.Callable; + +public class AirbyteSentry { + + @FunctionalInterface + public interface ThrowingRunnable { + + void call() throws Exception; + + } + + public static void runWithSpan(final String operation, final ThrowingRunnable command) throws Exception { + final ISpan span = Sentry.getSpan(); + final ISpan childSpan; + if (span == null) { + childSpan = Sentry.startTransaction("ROOT", operation); + } else { + childSpan = span.startChild(operation); + } + try { + command.call(); + childSpan.finish(SpanStatus.OK); + } catch (final Exception e) { + childSpan.setThrowable(e); + childSpan.finish(SpanStatus.INTERNAL_ERROR); + throw e; + } finally { + childSpan.finish(); + } + } + + public static T runWithSpan(final String operation, final Callable command) throws Exception { + final ISpan span = Sentry.getSpan(); + final ISpan childSpan; + if (span == null) { + childSpan = Sentry.startTransaction("ROOT", operation); + } else { + childSpan = span.startChild(operation); + } + try { + final T result = command.call(); + childSpan.finish(SpanStatus.OK); + return result; + } catch (final Exception e) { + childSpan.setThrowable(e); + childSpan.finish(SpanStatus.INTERNAL_ERROR); + throw e; + } finally { + childSpan.finish(); + } + } + +} diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 4d2d25c249ad0..c8619f21930df 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.1.63 +LABEL io.airbyte.version=0.1.65 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 4ed99e1c29b51..4ba4ae808d2b6 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -133,8 +133,8 @@ def transform_bigquery(config: Dict[str, Any]): "project": config["project_id"], "dataset": config["dataset_id"], "priority": config.get("transformation_priority", "interactive"), - "threads": 32, - "retries": 1, + "threads": 8, + "retries": 3, } if "credentials_json" in config: dbt_config["method"] = "service-account-json" @@ -161,7 +161,7 @@ def transform_postgres(config: Dict[str, Any]): "port": config["port"], "dbname": config["database"], "schema": config["schema"], - "threads": 32, + "threads": 8, } # if unset, we assume true. @@ -182,7 +182,7 @@ def transform_redshift(config: Dict[str, Any]): "port": config["port"], "dbname": config["database"], "schema": config["schema"], - "threads": 32, + "threads": 4, } return dbt_config @@ -202,9 +202,13 @@ def transform_snowflake(config: Dict[str, Any]): "database": config["database"].upper(), "warehouse": config["warehouse"].upper(), "schema": config["schema"].upper(), - "threads": 32, + "threads": 5, "client_session_keep_alive": False, "query_tag": "normalization", + "retry_all": True, + "retry_on_database_errors": True, + "connect_retries": 3, + "connect_timeout": 15, } return dbt_config @@ -259,7 +263,7 @@ def transform_mssql(config: Dict[str, Any]): "database": config["database"], "user": config["username"], "password": config["password"], - "threads": 32, + "threads": 8, # "authentication": "sql", # "trusted_connection": True, } diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py index cfd5c0e88cea7..bfab7943de5c6 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py @@ -147,8 +147,8 @@ def test_transform_bigquery(self): "priority": "interactive", "keyfile_json": {"type": "service_account-json"}, "location": "EU", - "retries": 1, - "threads": 32, + "retries": 3, + "threads": 8, } actual_keyfile = actual_output["keyfile_json"] @@ -167,8 +167,8 @@ def test_transform_bigquery_no_credentials(self): "project": "my_project_id", "dataset": "my_dataset_id", "priority": "interactive", - "retries": 1, - "threads": 32, + "retries": 3, + "threads": 8, } assert expected_output == actual_output @@ -192,7 +192,7 @@ def test_transform_postgres(self): "pass": "password123", "port": 5432, "schema": "public", - "threads": 32, + "threads": 8, "user": "a user", } @@ -225,7 +225,7 @@ def test_transform_postgres_ssh(self): "pass": "password123", "port": port, "schema": "public", - "threads": 32, + "threads": 8, "user": "a user", } @@ -252,7 +252,11 @@ def test_transform_snowflake(self): "query_tag": "normalization", "role": "AIRBYTE_ROLE", "schema": "AIRBYTE_SCHEMA", - "threads": 32, + "threads": 5, + "retry_all": True, + "retry_on_database_errors": True, + "connect_retries": 3, + "connect_timeout": 15, "type": "snowflake", "user": "AIRBYTE_USER", "warehouse": "AIRBYTE_WAREHOUSE", @@ -332,7 +336,7 @@ def test_transform(self): "pass": "password123", "port": 5432, "schema": "public", - "threads": 32, + "threads": 8, "user": "a user", } actual = TransformConfig().transform(DestinationType.postgres, input) diff --git a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java b/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java index afe97cfac9650..0ccd71d4435ad 100644 --- a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java +++ b/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java @@ -12,8 +12,6 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; import javax.annotation.Nullable; /** @@ -94,11 +92,6 @@ protected JsonNode getState() { } - @Override - protected List getRegexTests() throws Exception { - return new ArrayList<>(); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { // no-op, for now diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index a0fc565f73c9e..b90373ff4f2b4 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,13 +1,25 @@ # Changelog +## 0.1.46 +Fix `test_oneof_usage` test: [#9861](https://github.com/airbytehq/airbyte/pull/9861) + +## 0.1.45 +Check for not allowed keywords `allOf`, `not` in connectors schema: [#9851](https://github.com/airbytehq/airbyte/pull/9851) + +## 0.1.44 +Fix incorrect name of `primary_keys` attribute: [#9768](https://github.com/airbytehq/airbyte/pull/9768) + +## 0.1.43 +`TestFullRefresh` test can compare records using PKs: [#9768](https://github.com/airbytehq/airbyte/pull/9768) + ## 0.1.36 -Add assert that spec.json file does not have any `$ref` in it: [#8842](https://github.com/airbytehq/airbyte/pull/8842) +Add assert that `spec.json` file does not have any `$ref` in it: [#8842](https://github.com/airbytehq/airbyte/pull/8842) ## 0.1.32 -Add info about skipped failed tests in /test command message on GitHub: [#8691](https://github.com/airbytehq/airbyte/pull/8691) +Add info about skipped failed tests in `/test` command message on GitHub: [#8691](https://github.com/airbytehq/airbyte/pull/8691) ## 0.1.31 -Take ConfiguredAirbyteCatalog from discover command by default +Take `ConfiguredAirbyteCatalog` from discover command by default ## 0.1.30 Validate if each field in a stream has appeared at least once in some record. @@ -28,13 +40,13 @@ Add ignored fields for full refresh test Fix incorrect nested structures compare. ## 0.1.24 -Improve message about errors in the stream's schema: https://github.com/airbytehq/airbyte/pull/6934 +Improve message about errors in the stream's schema: [#6934](https://github.com/airbytehq/airbyte/pull/6934) ## 0.1.23 Fix incorrect auth init flow check defect. ## 0.1.22 -Fix checking schemas with root $ref keyword +Fix checking schemas with root `$ref` keyword ## 0.1.21 Fix rootObject oauth init parameter check @@ -49,22 +61,22 @@ Assert a non-empty overlap between the fields present in the record and the decl Fix checking date-time format against nullable field. ## 0.1.17 -Fix serialize function for acceptance-tests: https://github.com/airbytehq/airbyte/pull/5738 +Fix serialize function for acceptance-tests: [#5738](https://github.com/airbytehq/airbyte/pull/5738) ## 0.1.16 -Fix for flake8-ckeck for acceptance-tests: https://github.com/airbytehq/airbyte/pull/5785 +Fix for flake8-ckeck for acceptance-tests: [#5785](https://github.com/airbytehq/airbyte/pull/5785) ## 0.1.15 -Add detailed logging for acceptance tests: https://github.com/airbytehq/airbyte/pull/5392 +Add detailed logging for acceptance tests: [5392](https://github.com/airbytehq/airbyte/pull/5392) ## 0.1.14 -Fix for NULL datetime in MySQL format (i.e. 0000-00-00): https://github.com/airbytehq/airbyte/pull/4465 +Fix for NULL datetime in MySQL format (i.e. `0000-00-00`): [#4465](https://github.com/airbytehq/airbyte/pull/4465) ## 0.1.13 -Replace `validate_output_from_all_streams` with `empty_streams` param: https://github.com/airbytehq/airbyte/pull/4897 +Replace `validate_output_from_all_streams` with `empty_streams` param: [#4897](https://github.com/airbytehq/airbyte/pull/4897) ## 0.1.12 -Improve error message when data mismatches schema: https://github.com/airbytehq/airbyte/pull/4753 +Improve error message when data mismatches schema: [#4753](https://github.com/airbytehq/airbyte/pull/4753) ## 0.1.11 Fix error in the naming of method `test_match_expected` for class `TestSpec`. @@ -73,19 +85,20 @@ Fix error in the naming of method `test_match_expected` for class `TestSpec`. Add validation of input config.json against spec.json. ## 0.1.9 -Add configurable validation of schema for all records in BasicRead test: https://github.com/airbytehq/airbyte/pull/4345 +Add configurable validation of schema for all records in BasicRead test: [#4345](https://github.com/airbytehq/airbyte/pull/4345) + The validation is ON by default. To disable validation for the source you need to set `validate_schema: off` in the config file. ## 0.1.8 -Fix cursor_path to support nested and absolute paths: https://github.com/airbytehq/airbyte/pull/4552 +Fix cursor_path to support nested and absolute paths: [#4552](https://github.com/airbytehq/airbyte/pull/4552) ## 0.1.7 Add: `test_spec` additionally checks if Dockerfile has `ENV AIRBYTE_ENTRYPOINT` defined and equal to space_joined `ENTRYPOINT` ## 0.1.6 -Add test whether PKs present and not None if `source_defined_primary_key` defined: https://github.com/airbytehq/airbyte/pull/4140 +Add test whether PKs present and not None if `source_defined_primary_key` defined: [#4140](https://github.com/airbytehq/airbyte/pull/4140) ## 0.1.5 -Add configurable timeout for the acceptance tests: https://github.com/airbytehq/airbyte/pull/4296 +Add configurable timeout for the acceptance tests: [#4296](https://github.com/airbytehq/airbyte/pull/4296) diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index ad60ff025240f..016e23dbed02d 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.1.42 +LABEL io.airbyte.version=0.1.46 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/pytest.ini b/airbyte-integrations/bases/source-acceptance-test/pytest.ini index dbd64386989f6..e1827862065ae 100644 --- a/airbyte-integrations/bases/source-acceptance-test/pytest.ini +++ b/airbyte-integrations/bases/source-acceptance-test/pytest.ini @@ -3,3 +3,6 @@ addopts = -r fEsx --capture=no -vv --log-level=INFO --color=yes --force-sugar testpaths = source_acceptance_test/tests + +markers = + default_timeout diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py index 6ecdaacbf9972..8e0468ee8193d 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + from .base import BaseTest __all__ = ["BaseTest"] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/__init__.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/__init__.py index 18099f11143fb..cc93282d56608 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/__init__.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/__init__.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + from .test_core import TestBasicRead, TestConnection, TestDiscovery, TestSpec from .test_full_refresh import TestFullRefresh from .test_incremental import TestIncremental diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index adffd750e0eaf..c1d1da13a4785 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -11,56 +11,89 @@ from typing import Any, Dict, List, Mapping, MutableMapping, Set import dpath.util +import jsonschema import pytest -from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status, Type +from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status, Type from docker.errors import ContainerError -from jsonschema import validate from jsonschema._utils import flatten from source_acceptance_test.base import BaseTest from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig from source_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, make_hashable, verify_records_schema -from source_acceptance_test.utils.common import find_key_inside_schema +from source_acceptance_test.utils.common import find_key_inside_schema, find_keyword_schema from source_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure +@pytest.fixture(name="connector_spec_dict") +def connector_spec_dict_fixture(actual_connector_spec): + return json.loads(actual_connector_spec.json()) + + +@pytest.fixture(name="actual_connector_spec") +def actual_connector_spec_fixture(request: BaseTest, docker_runner): + if not request.instance.spec_cache: + output = docker_runner.call_spec() + spec_messages = filter_output(output, Type.SPEC) + assert len(spec_messages) == 1, "Spec message should be emitted exactly once" + spec = spec_messages[0].spec + request.spec_cache = spec + return request.spec_cache + + @pytest.mark.default_timeout(10) class TestSpec(BaseTest): spec_cache: ConnectorSpecification = None - @pytest.fixture(name="actual_connector_spec") - def actual_connector_spec_fixture(request: BaseTest, docker_runner): - if not request.spec_cache: - output = docker_runner.call_spec() - spec_messages = filter_output(output, Type.SPEC) - assert len(spec_messages) == 1, "Spec message should be emitted exactly once" - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT"), "AIRBYTE_ENTRYPOINT must be set in dockerfile" - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT") == " ".join( - docker_runner.entry_point - ), "env should be equal to space-joined entrypoint" - spec = spec_messages[0].spec - request.spec_cache = spec - return request.spec_cache - - @pytest.fixture(name="connector_spec_dict") - def connector_spec_dict_fixture(request: BaseTest, actual_connector_spec): - return json.loads(actual_connector_spec.json()) - - def test_match_expected( - self, connector_spec: ConnectorSpecification, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict - ): + def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict): + """Check that config matches the actual schema from the spec call""" + # Getting rid of technical variables that start with an underscore + config = {key: value for key, value in connector_config.data.items() if not key.startswith("_")} + + try: + jsonschema.validate(instance=config, schema=actual_connector_spec.connectionSpecification) + except jsonschema.exceptions.ValidationError as err: + pytest.fail(f"Config invalid: {err}") + except jsonschema.exceptions.SchemaError as err: + pytest.fail(f"Spec is invalid: {err}") + def test_match_expected(self, connector_spec: ConnectorSpecification, actual_connector_spec: ConnectorSpecification): + """Check that spec call returns a spec equals to expected one""" if connector_spec: assert actual_connector_spec == connector_spec, "Spec should be equal to the one in spec.json file" - # Getting rid of technical variables that start with an underscore - config = {key: value for key, value in connector_config.data.items() if not key.startswith("_")} - spec_message_schema = actual_connector_spec.connectionSpecification - validate(instance=config, schema=spec_message_schema) + def test_docker_env(self, actual_connector_spec: ConnectorSpecification, docker_runner: ConnectorRunner): + """Check that connector's docker image has required envs""" + assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT"), "AIRBYTE_ENTRYPOINT must be set in dockerfile" + assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT") == " ".join( + docker_runner.entry_point + ), "env should be equal to space-joined entrypoint" + + def test_oneof_usage(self, actual_connector_spec: ConnectorSpecification): + """Check that if spec contains oneOf it follows the rules according to reference + https://docs.airbyte.io/connector-development/connector-specification-reference + """ + docs_url = "https://docs.airbyte.io/connector-development/connector-specification-reference" + docs_msg = f"See specification reference at {docs_url}." + + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) + variant_paths = schema_helper.find_nodes(keys=["oneOf", "anyOf"]) + + for variant_path in variant_paths: + top_level_obj = schema_helper.get_node(variant_path[:-1]) + assert ( + top_level_obj.get("type") == "object" + ), f"The top-level definition in a `oneOf` block should have type: object. misconfigured object: {top_level_obj}. {docs_msg}" - js_helper = JsonSchemaHelper(spec_message_schema) - variants = js_helper.find_variant_paths() - js_helper.validate_variant_paths(variants) + variants = schema_helper.get_node(variant_path) + for variant in variants: + assert "properties" in variant, f"Each item in the oneOf array should be a property with type object. {docs_msg}" + + variant_props = [set(list(v["properties"].keys())) for v in variants] + common_props = set.intersection(*variant_props) + assert common_props, "There should be at least one common property for oneOf subobjects" + assert any( + [all(["const" in var["properties"][prop] for var in variants]) for prop in common_props] + ), f"Any of {common_props} properties in {'.'.join(variant_path)} has no const keyword. {docs_msg}" def test_required(self): """Check that connector will fail if any required field is missing""" @@ -81,17 +114,13 @@ def test_defined_refs_exist_in_json_spec_file(self, connector_spec_dict: dict): assert not check_result, "Found unresolved `$refs` value in spec.json file" def test_oauth_flow_parameters(self, actual_connector_spec: ConnectorSpecification): + """Check if connector has correct oauth flow parameters according to + https://docs.airbyte.io/connector-development/connector-specification-reference """ - Check if connector has correct oauth flow parameters according to https://docs.airbyte.io/connector-development/connector-specification-reference - """ - self._validate_authflow_parameters(actual_connector_spec) - - @staticmethod - def _validate_authflow_parameters(connector_spec: ConnectorSpecification): - if not connector_spec.authSpecification: + if not actual_connector_spec.authSpecification: return - spec_schema = connector_spec.connectionSpecification - oauth_spec = connector_spec.authSpecification.oauth2Specification + spec_schema = actual_connector_spec.connectionSpecification + oauth_spec = actual_connector_spec.authSpecification.oauth2Specification parameters: List[List[str]] = oauth_spec.oauthFlowInitParameters + oauth_spec.oauthFlowOutputParameters root_object = oauth_spec.rootObject if len(root_object) == 0: @@ -104,7 +133,7 @@ def _validate_authflow_parameters(connector_spec: ConnectorSpecification): params = {"/" + "/".join([f"{root_object[0]}({root_object[1]})", *p]) for p in parameters} schema_path = set(get_expected_schema_structure(spec_schema, annotate_one_of=True)) else: - assert "rootObject cannot have more than 2 elements" + pytest.fail("rootObject cannot have more than 2 elements") diff = params - schema_path assert diff == set(), f"Specified oauth fields are missed from spec schema: {diff}" @@ -136,34 +165,30 @@ def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runn @pytest.mark.default_timeout(30) class TestDiscovery(BaseTest): def test_discover(self, connector_config, docker_runner: ConnectorRunner): + """Verify that discover produce correct schema.""" output = docker_runner.call_discover(config=connector_config) catalog_messages = filter_output(output, Type.CATALOG) assert len(catalog_messages) == 1, "Catalog message should be emitted exactly once" - # TODO(sherifnada) return this once an input bug is fixed (test suite currently fails if this file is not provided) - # if catalog: - # for stream1, stream2 in zip(catalog_messages[0].catalog.streams, catalog.streams): - # assert stream1.json_schema == stream2.json_schema, f"Streams: {stream1.name} vs {stream2.name}, stream schemas should match" - # stream1.json_schema = None - # stream2.json_schema = None - # assert stream1.dict() == stream2.dict(), f"Streams {stream1.name} and {stream2.name}, stream configs should match" - - def test_defined_cursors_exist_in_schema(self, connector_config, discovered_catalog): - """ - Check if all of the source defined cursor fields are exists on stream's json schema. - """ + assert catalog_messages[0].catalog, "Message should have catalog" + assert catalog_messages[0].catalog.streams, "Catalog should contain streams" + + def test_defined_cursors_exist_in_schema(self, discovered_catalog: Mapping[str, Any]): + """Check if all of the source defined cursor fields are exists on stream's json schema.""" for stream_name, stream in discovered_catalog.items(): - if stream.default_cursor_field: - schema = stream.json_schema - assert "properties" in schema, "Top level item should have an 'object' type for {stream_name} stream schema" - properties = schema["properties"] - cursor_path = "/properties/".join(stream.default_cursor_field) - assert dpath.util.search( - properties, cursor_path - ), f"Some of defined cursor fields {stream.default_cursor_field} are not specified in discover schema properties for {stream_name} stream" - - def test_defined_refs_exist_in_schema(self, connector_config, discovered_catalog): - """Checking for the presence of unresolved `$ref`s values within each json schema""" + if not stream.default_cursor_field: + continue + schema = stream.json_schema + assert "properties" in schema, f"Top level item should have an 'object' type for {stream_name} stream schema" + cursor_path = "/properties/".join(stream.default_cursor_field) + cursor_field_location = dpath.util.search(schema["properties"], cursor_path) + assert cursor_field_location, ( + f"Some of defined cursor fields {stream.default_cursor_field} are not specified in discover schema " + f"properties for {stream_name} stream" + ) + + def test_defined_refs_exist_in_schema(self, discovered_catalog: Mapping[str, Any]): + """Check the presence of unresolved `$ref`s values within each json schema.""" schemas_errors = [] for stream_name, stream in discovered_catalog.items(): check_result = find_key_inside_schema(schema_item=stream.json_schema, key="$ref") @@ -172,6 +197,26 @@ def test_defined_refs_exist_in_schema(self, connector_config, discovered_catalog assert not schemas_errors, f"Found unresolved `$refs` values for selected streams: {tuple(schemas_errors)}." + @pytest.mark.parametrize("keyword", ["allOf", "not"]) + def test_defined_keyword_exist_in_schema(self, keyword, discovered_catalog): + """Checking for the presence of not allowed keywords within each json schema""" + schemas_errors = [] + for stream_name, stream in discovered_catalog.items(): + check_result = find_keyword_schema(stream.json_schema, key=keyword) + if check_result: + schemas_errors.append(stream_name) + + assert not schemas_errors, f"Found not allowed `{keyword}` keyword for selected streams: {schemas_errors}." + + def test_primary_keys_exist_in_schema(self, discovered_catalog: Mapping[str, Any]): + """Check that all primary keys are present in catalog.""" + for stream_name, stream in discovered_catalog.items(): + for pk in stream.source_defined_primary_key or []: + schema = stream.json_schema + pk_path = "/properties/".join(pk) + pk_field_location = dpath.util.search(schema["properties"], pk_path) + assert pk_field_location, f"One of the PKs ({pk}) is not specified in discover schema for {stream_name} stream" + def primary_keys_for_records(streams, records): streams_with_primary_key = [stream for stream in streams if stream.stream.source_defined_primary_key] @@ -191,7 +236,7 @@ class TestBasicRead(BaseTest): @staticmethod def _validate_records_structure(records: List[AirbyteRecordMessage], configured_catalog: ConfiguredAirbyteCatalog): """ - Check object structure simmilar to one expected by schema. Sometimes + Check object structure similar to one expected by schema. Sometimes just running schema validation is not enough case schema could have additionalProperties parameter set to true and no required fields therefore any arbitrary object would pass schema validation. @@ -283,7 +328,7 @@ def _validate_field_appears_at_least_once(self, records: List, configured_catalo assert not stream_name_to_empty_fields_mapping, msg def _validate_expected_records( - self, records: List[AirbyteMessage], expected_records: List[AirbyteMessage], flags, detailed_logger: Logger + self, records: List[AirbyteRecordMessage], expected_records: List[AirbyteRecordMessage], flags, detailed_logger: Logger ): """ We expect some records from stream to match expected_records, partially or fully, in exact or any order. @@ -312,7 +357,7 @@ def test_read( connector_config, configured_catalog, inputs: BasicReadTestConfig, - expected_records: List[AirbyteMessage], + expected_records: List[AirbyteRecordMessage], docker_runner: ConnectorRunner, detailed_logger, ): @@ -327,9 +372,9 @@ def test_read( self._validate_empty_streams(records=records, configured_catalog=configured_catalog, allowed_empty_streams=inputs.empty_streams) for pks, record in primary_keys_for_records(streams=configured_catalog.streams, records=records): for pk_path, pk_value in pks.items(): - assert pk_value is not None, ( - f"Primary key subkeys {repr(pk_path)} " f"have null values or not present in {record.stream} stream records." - ) + assert ( + pk_value is not None + ), f"Primary key subkeys {repr(pk_path)} have null values or not present in {record.stream} stream records." # TODO: remove this condition after https://github.com/airbytehq/airbyte/issues/8312 is done if inputs.validate_data_points: @@ -358,8 +403,8 @@ def remove_extra_fields(record: Any, spec: Any) -> Any: @staticmethod def compare_records( stream_name: str, - actual: List[Dict[str, Any]], - expected: List[Dict[str, Any]], + actual: List[Mapping[str, Any]], + expected: List[Mapping[str, Any]], extra_fields: bool, exact_order: bool, extra_records: bool, @@ -394,7 +439,7 @@ def compare_records( pytest.fail(msg) @staticmethod - def group_by_stream(records) -> MutableMapping[str, List[MutableMapping]]: + def group_by_stream(records: List[AirbyteRecordMessage]) -> MutableMapping[str, List[MutableMapping]]: """Group records by a source stream""" result = defaultdict(list) for record in records: diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py index 909ffaf071cfb..9b2c3c2795439 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py @@ -5,12 +5,33 @@ from collections import defaultdict from functools import partial from logging import Logger +from typing import List, Mapping import pytest from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type from source_acceptance_test.base import BaseTest from source_acceptance_test.config import ConnectionTestConfig -from source_acceptance_test.utils import ConnectorRunner, SecretDict, full_refresh_only_catalog, make_hashable +from source_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, full_refresh_only_catalog, make_hashable +from source_acceptance_test.utils.json_schema_helper import CatalogField + + +def primary_keys_by_stream(configured_catalog: ConfiguredAirbyteCatalog) -> Mapping[str, List[CatalogField]]: + """Get PK fields for each stream + + :param configured_catalog: + :return: + """ + data = {} + for stream in configured_catalog.streams: + helper = JsonSchemaHelper(schema=stream.stream.json_schema) + pks = stream.primary_key or [] + data[stream.stream.name] = [helper.field(pk) for pk in pks] + + return data + + +def primary_keys_only(record, pks): + return ";".join([f"{pk.path}={pk.parse(record)}" for pk in pks]) @pytest.mark.default_timeout(20 * 60) @@ -37,8 +58,13 @@ def test_sequential_reads( for record in records_2: records_by_stream_2[record.stream].append(record.data) + pks_by_stream = primary_keys_by_stream(configured_catalog) + for stream in records_by_stream_1: - serializer = partial(make_hashable, exclude_fields=ignored_fields.get(stream)) + if pks_by_stream.get(stream): + serializer = partial(primary_keys_only, pks=pks_by_stream.get(stream)) + else: + serializer = partial(make_hashable, exclude_fields=ignored_fields.get(stream)) stream_records_1 = records_by_stream_1.get(stream) stream_records_2 = records_by_stream_2.get(stream) # Using diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py index 52da161c15e4f..437d17a81390d 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py @@ -81,3 +81,24 @@ def find_key_inside_schema(schema_item: Union[dict, list, str], key: str = "$ref item = find_key_inside_schema(schema_object_value, key) if item is not None: return item + + +def find_keyword_schema(schema: Union[dict, list, str], key: str) -> bool: + """Find at least one keyword in a schema, skip object properties""" + + def _find_keyword(schema, key, _skip=False): + if isinstance(schema, list): + for v in schema: + _find_keyword(v, key) + elif isinstance(schema, dict): + for k, v in schema.items(): + if k == key and not _skip: + raise StopIteration + rec_skip = k == "properties" and schema.get("type") == "object" + _find_keyword(v, key, rec_skip) + + try: + _find_keyword(schema, key) + except StopIteration: + return True + return False diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py index dbc805da34d41..c7613756b6cb9 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py @@ -6,13 +6,14 @@ from functools import reduce from typing import Any, List, Mapping, Optional, Set -import dpath.util import pendulum from jsonref import JsonRef class CatalogField: - """Field class to represent cursor/pk fields""" + """Field class to represent cursor/pk fields. + It eases the read of values from records according to schema definition. + """ def __init__(self, schema: Mapping[str, Any], path: List[str]): self.schema = schema @@ -49,16 +50,46 @@ def parse(self, record: Mapping[str, Any], path: Optional[List[str]] = None) -> class JsonSchemaHelper: + """Helper class to simplify schema validation and read of records according to their schema.""" + def __init__(self, schema): self._schema = schema - def get_ref(self, path: List[str]): + def get_ref(self, path: str) -> Any: + """Resolve reference + + :param path: reference (#/definitions/SomeClass, etc) + :return: part of schema that is definition of the reference + :raises KeyError: in case path can't be followed + """ node = self._schema for segment in path.split("/")[1:]: node = node[segment] return node def get_property(self, path: List[str]) -> Mapping[str, Any]: + """Get any part of schema according to provided path, resolves $refs if necessary + + schema = { + "properties": { + "field1": { + "properties": { + "nested_field": { + + } + } + }, + "field2": ... + } + } + + helper = JsonSchemaHelper(schema) + helper.get_property(["field1", "nested_field"]) == + + :param path: list of fields in the order of navigation + :return: discovered part of schema + :raises KeyError: in case path can't be followed + """ node = self._schema for segment in path: if "$ref" in node: @@ -67,16 +98,40 @@ def get_property(self, path: List[str]) -> Mapping[str, Any]: return node def field(self, path: List[str]) -> CatalogField: + """Get schema property and wrap it into CatalogField. + + CatalogField is a helper to ease the read of values from records according to schema definition. + + :param path: list of fields in the order of navigation + :return: discovered part of schema wrapped in CatalogField + :raises KeyError: in case path can't be followed + """ return CatalogField(schema=self.get_property(path), path=path) - def find_variant_paths(self) -> List[List[str]]: + def get_node(self, path: List[str]) -> Any: + """Return part of schema by specified path + + :param path: list of fields in the order of navigation """ - return list of json object paths for oneOf or anyOf attributes + + node = self._schema + for segment in path: + if "$ref" in node: + node = self.get_ref(node["$ref"]) + node = node[segment] + return node + + def find_nodes(self, keys: List[str]) -> List[List[str]]: + """Get all nodes of schema that has specifies properties + + :param keys: + :return: list of json object paths """ variant_paths = [] - def traverse_schema(_schema, path=[]): - if path and path[-1] in ["oneOf", "anyOf"]: + def traverse_schema(_schema, path=None): + path = path or [] + if path and path[-1] in keys: variant_paths.append(path) for item in _schema: next_obj = _schema[item] if isinstance(_schema, dict) else item @@ -86,51 +141,6 @@ def traverse_schema(_schema, path=[]): traverse_schema(self._schema) return variant_paths - def validate_variant_paths(self, variant_paths: List[List[str]]): - """ - Validate oneOf paths according to reference - https://docs.airbyte.io/connector-development/connector-specification-reference - """ - - def get_top_level_item(variant_path: List[str]): - # valid path should contain at least 3 items - path_to_schema_obj = variant_path[:-1] - return dpath.util.get(self._schema, "/".join(path_to_schema_obj)) - - for variant_path in variant_paths: - top_level_obj = get_top_level_item(variant_path) - if "$ref" in top_level_obj: - obj_def = top_level_obj["$ref"].split("/")[-1] - top_level_obj = self._schema["definitions"][obj_def] - """ - 1. The top-level item containing the oneOf must have type: object - """ - assert ( - top_level_obj.get("type") == "object" - ), f"The top-level definition in a `oneOf` block should have type: object. misconfigured object: {top_level_obj}. See specification reference at https://docs.airbyte.io/connector-development/connector-specification-reference" - """ - - 2. Each item in the oneOf array must be a property with type: object - """ - variants = dpath.util.get(self._schema, "/".join(variant_path)) - for variant in variants: - assert ( - "properties" in variant - ), "Each item in the oneOf array should be a property with type object. See specification reference at https://docs.airbyte.io/connector-development/connector-specification-reference" - - """ - 3. One string field with the same property name must be - consistently present throughout each object inside the oneOf - array. It is required to add a const value unique to that oneOf - option. - """ - variant_props = [set(list(v["properties"].keys())) for v in variants] - common_props = set.intersection(*variant_props) - assert common_props, "There should be at least one common property for oneOf subobjects" - assert any( - [all(["const" in var["properties"][prop] for var in variants]) for prop in common_props] - ), f"Any of {common_props} properties in {'.'.join(variant_path)} has no const keyword. See specification reference at https://docs.airbyte.io/connector-development/connector-specification-reference" - def get_object_structure(obj: dict) -> List[str]: """ diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/__init__.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/__init__.py index e69de29bb2d1d..46b7376756ec6 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/__init__.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py index becae418a4372..61e36b4db89cd 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py @@ -5,140 +5,10 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import ( - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - ConnectorSpecification, - Type, -) +from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Type from source_acceptance_test.config import BasicReadTestConfig from source_acceptance_test.tests.test_core import TestBasicRead as _TestBasicRead from source_acceptance_test.tests.test_core import TestDiscovery as _TestDiscovery -from source_acceptance_test.tests.test_core import TestSpec as _TestSpec - - -@pytest.mark.parametrize( - "connector_spec, should_fail", - [ - ( - { - "connectionSpecification": { - "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - "$ref": None, - }, - } - }, - True, - ), - ( - { - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "properties": {"refresh_token": {"type": "string"}, "$ref": None}, - } - }, - } - }, - True, - ), - ( - { - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_server_input_specification": { - "type": "object", - "properties": {"refresh_token": {"type": "string"}, "$ref": None}, - } - }, - } - }, - True, - ), - ( - { - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_server_output_specification": { - "type": "object", - "properties": {"refresh_token": {"type": "string"}, "$ref": None}, - } - }, - } - }, - True, - ), - ( - { - "connectionSpecification": { - "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - }, - } - }, - False, - ), - ( - { - "connectionSpecification": { - "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - }, - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_server_output_specification": { - "type": "object", - "properties": {"refresh_token": {"type": "string"}}, - } - }, - }, - }, - False, - ), - ({"$ref": None}, True), - ({"properties": {"user": {"$ref": None}}}, True), - ({"properties": {"user": {"$ref": "user.json"}}}, True), - ({"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, False), - ({"properties": {"fake_items": {"type": "array", "items": {"$ref": "fake_item.json"}}}}, True), - ], -) -def test_ref_in_spec_schemas(connector_spec, should_fail): - t = _TestSpec() - if should_fail is True: - with pytest.raises(AssertionError): - t.test_defined_refs_exist_in_json_spec_file(connector_spec_dict=connector_spec) - else: - t.test_defined_refs_exist_in_json_spec_file(connector_spec_dict=connector_spec) @pytest.mark.parametrize( @@ -159,9 +29,9 @@ def test_discovery(schema, cursors, should_fail): } if should_fail: with pytest.raises(AssertionError): - t.test_defined_cursors_exist_in_schema(None, discovered_catalog) + t.test_defined_cursors_exist_in_schema(discovered_catalog) else: - t.test_defined_cursors_exist_in_schema(None, discovered_catalog) + t.test_defined_cursors_exist_in_schema(discovered_catalog) @pytest.mark.parametrize( @@ -190,9 +60,49 @@ def test_ref_in_discovery_schemas(schema, should_fail): discovered_catalog = {"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": schema})} if should_fail: with pytest.raises(AssertionError): - t.test_defined_refs_exist_in_schema(None, discovered_catalog) + t.test_defined_refs_exist_in_schema(discovered_catalog) + else: + t.test_defined_refs_exist_in_schema(discovered_catalog) + + +@pytest.mark.parametrize( + "schema, keyword, should_fail", + [ + ({}, "allOf", False), + ({"allOf": [{"type": "string"}, {"maxLength": 1}]}, "allOf", True), + ({"type": "object", "properties": {"allOf": {"type": "string"}}}, "allOf", False), + ({"type": "object", "properties": {"name": {"allOf": [{"type": "string"}, {"maxLength": 1}]}}}, "allOf", True), + ( + {"type": "object", "properties": {"name": {"type": "array", "items": {"allOf": [{"type": "string"}, {"maxLength": 4}]}}}}, + "allOf", + True, + ), + ( + { + "type": "object", + "properties": { + "name": { + "type": "array", + "items": {"anyOf": [{"type": "number"}, {"allOf": [{"type": "string"}, {"maxLength": 4}, {"minLength": 2}]}]}, + } + }, + }, + "allOf", + True, + ), + ({"not": {"type": "string"}}, "not", True), + ({"type": "object", "properties": {"not": {"type": "string"}}}, "not", False), + ({"type": "object", "properties": {"name": {"not": {"type": "string"}}}}, "not", True), + ], +) +def test_keyword_in_discovery_schemas(schema, keyword, should_fail): + t = _TestDiscovery() + discovered_catalog = {"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": schema})} + if should_fail: + with pytest.raises(AssertionError): + t.test_defined_keyword_exist_in_schema(keyword, discovered_catalog) else: - t.test_defined_refs_exist_in_schema(None, discovered_catalog) + t.test_defined_keyword_exist_in_schema(keyword, discovered_catalog) @pytest.mark.parametrize( @@ -234,226 +144,6 @@ def test_read(schema, record, should_fail): t.test_read(None, catalog, input_config, [], docker_runner_mock, MagicMock()) -@pytest.mark.parametrize( - "connector_spec, expected_error", - [ - # SUCCESS: no authSpecification specified - (ConnectorSpecification(connectionSpecification={}), ""), - # FAIL: Field specified in root object does not exist - ( - ConnectorSpecification( - connectionSpecification={"type": "object"}, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "Specified oauth fields are missed from spec schema:", - ), - # SUCCESS: Empty root object - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - }, - }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "", - ), - # FAIL: Some oauth fields missed - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "credentials": { - "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - }, - } - }, - }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "Specified oauth fields are missed from spec schema:", - ), - # SUCCESS: case w/o oneOf property - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "credentials": { - "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - }, - } - }, - }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "", - ), - # SUCCESS: case w/ oneOf property - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "credentials": { - "type": "object", - "oneOf": [ - { - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - } - }, - { - "properties": { - "api_key": {"type": "string"}, - } - }, - ], - } - }, - }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "", - ), - # FAIL: Wrong root object index - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "credentials": { - "type": "object", - "oneOf": [ - { - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - } - }, - { - "properties": { - "api_key": {"type": "string"}, - } - }, - ], - } - }, - }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 1], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "Specified oauth fields are missed from spec schema:", - ), - # SUCCESS: root object index equal to 1 - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "credentials": { - "type": "object", - "oneOf": [ - { - "properties": { - "api_key": {"type": "string"}, - } - }, - { - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - } - }, - ], - } - }, - }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 1], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "", - ), - ], -) -def test_validate_oauth_flow(connector_spec, expected_error): - t = _TestSpec() - if expected_error: - with pytest.raises(AssertionError, match=expected_error): - t.test_oauth_flow_parameters(connector_spec) - else: - t.test_oauth_flow_parameters(connector_spec) - - @pytest.mark.parametrize( "records, configured_catalog, expected_error", [ diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py index 11478abed4852..0cf64c67ea92b 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py @@ -122,20 +122,6 @@ def test_absolute_path(records, stream_mapping, singer_state): assert state_value == pendulum.datetime(2014, 1, 1, 22, 3, 11), "state value must be correctly found" -def test_json_schema_helper_mssql(mssql_spec_schema): - js_helper = JsonSchemaHelper(mssql_spec_schema) - variant_paths = js_helper.find_variant_paths() - assert variant_paths == [["properties", "ssl_method", "oneOf"]] - js_helper.validate_variant_paths(variant_paths) - - -def test_json_schema_helper_postgres(postgres_source_spec_schema): - js_helper = JsonSchemaHelper(postgres_source_spec_schema) - variant_paths = js_helper.find_variant_paths() - assert variant_paths == [["properties", "replication_method", "oneOf"]] - js_helper.validate_variant_paths(variant_paths) - - def test_json_schema_helper_pydantic_generated(): class E(str, Enum): A = "dda" @@ -162,7 +148,7 @@ class Root(BaseModel): f: Union[A, B] js_helper = JsonSchemaHelper(Root.schema()) - variant_paths = js_helper.find_variant_paths() + variant_paths = js_helper.find_nodes(keys=["anyOf", "oneOf"]) assert len(variant_paths) == 2 assert variant_paths == [["properties", "f", "anyOf"], ["definitions", "C", "properties", "e", "anyOf"]] # TODO: implement validation for pydantic generated objects as well diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_spec.py new file mode 100644 index 0000000000000..4ea6a67229e1b --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_spec.py @@ -0,0 +1,485 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Callable, Dict + +import pytest +from airbyte_cdk.models import ConnectorSpecification +from source_acceptance_test.tests.test_core import TestSpec as _TestSpec + + +@pytest.mark.parametrize( + "connector_spec, should_fail", + [ + ( + { + "connectionSpecification": { + "type": "object", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "$ref": None, + }, + } + }, + True, + ), + ( + { + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": {"refresh_token": {"type": "string"}, "$ref": None}, + } + }, + } + }, + True, + ), + ( + { + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"refresh_token": {"type": "string"}, "$ref": None}, + } + }, + } + }, + True, + ), + ( + { + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_server_output_specification": { + "type": "object", + "properties": {"refresh_token": {"type": "string"}, "$ref": None}, + } + }, + } + }, + True, + ), + ( + { + "connectionSpecification": { + "type": "object", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + }, + } + }, + False, + ), + ( + { + "connectionSpecification": { + "type": "object", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + }, + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_server_output_specification": { + "type": "object", + "properties": {"refresh_token": {"type": "string"}}, + } + }, + }, + }, + False, + ), + ({"$ref": None}, True), + ({"properties": {"user": {"$ref": None}}}, True), + ({"properties": {"user": {"$ref": "user.json"}}}, True), + ({"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, False), + ({"properties": {"fake_items": {"type": "array", "items": {"$ref": "fake_item.json"}}}}, True), + ], +) +def test_ref_in_spec_schemas(connector_spec, should_fail): + t = _TestSpec() + if should_fail is True: + with pytest.raises(AssertionError): + t.test_defined_refs_exist_in_json_spec_file(connector_spec_dict=connector_spec) + else: + t.test_defined_refs_exist_in_json_spec_file(connector_spec_dict=connector_spec) + + +def parametrize_test_case(*test_cases: Dict[str, Any]) -> Callable: + """Util to wrap pytest.mark.parametrize and provider more friendlier interface. + + @parametrize_test_case({"value": 10, "expected_to_fail": True}, {"value": 100, "expected_to_fail": False}) + + an equivalent to: + + @pytest.mark.parametrize("value,expected_to_fail", [(10, True), (100, False)]) + + :param test_cases: list of dicts + :return: pytest.mark.parametrize decorator + """ + all_keys = set() + for test_case in test_cases: + all_keys = all_keys.union(set(test_case.keys())) + all_keys.discard("test_id") + + test_ids = [] + values = [] + for test_case in test_cases: + test_ids.append(test_case.pop("test_id", None)) + values.append(tuple(test_case.get(k) for k in all_keys)) + + return pytest.mark.parametrize(",".join(all_keys), values, ids=test_ids) + + +@parametrize_test_case( + { + "test_id": "all_good", + "connector_spec": { + "type": "object", + "properties": { + "select_type": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "first option"}, + "something": {"type": "string"}, + }, + }, + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "second option"}, + "some_field": {"type": "boolean"}, + }, + }, + ], + }, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + }, + }, + "should_fail": False, + }, + { + "test_id": "top_level_node_is_not_of_object_type", + "connector_spec": { + "type": "object", + "properties": { + "select_type": { + "oneOf": [], + }, + }, + }, + "should_fail": True, + }, + { + "test_id": "all_oneof_options_should_have_same_constant_attribute", + "connector_spec": { + "type": "object", + "properties": { + "select_type": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "wrong_title": {"type": "string", "title": "Title", "const": "first option"}, + "something": {"type": "string"}, + }, + }, + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "second option"}, + "some_field": {"type": "boolean"}, + }, + }, + ], + }, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + }, + }, + "should_fail": True, + }, + { + "test_id": "one_of_item_is_not_of_type_object", + "connector_spec": { + "type": "object", + "properties": { + "select_type": { + "type": "object", + "oneOf": [ + { + "type": "string", + }, + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "second option"}, + "some_field": {"type": "boolean"}, + }, + }, + ], + }, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + }, + }, + "should_fail": True, + }, +) +def test_oneof_usage(connector_spec, should_fail): + t = _TestSpec() + if should_fail is True: + with pytest.raises(AssertionError): + t.test_oneof_usage(actual_connector_spec=ConnectorSpecification(connectionSpecification=connector_spec)) + else: + t.test_oneof_usage(actual_connector_spec=ConnectorSpecification(connectionSpecification=connector_spec)) + + +@pytest.mark.parametrize( + "connector_spec, expected_error", + [ + # SUCCESS: no authSpecification specified + (ConnectorSpecification(connectionSpecification={}), ""), + # FAIL: Field specified in root object does not exist + ( + ConnectorSpecification( + connectionSpecification={"type": "object"}, + authSpecification={ + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", 0], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + }, + }, + ), + "Specified oauth fields are missed from spec schema:", + ), + # SUCCESS: Empty root object + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + }, + }, + authSpecification={ + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": [], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + }, + }, + ), + "", + ), + # FAIL: Some oauth fields missed + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "type": "object", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + }, + } + }, + }, + authSpecification={ + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", 0], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + }, + }, + ), + "Specified oauth fields are missed from spec schema:", + ), + # SUCCESS: case w/o oneOf property + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "type": "object", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + }, + } + }, + }, + authSpecification={ + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials"], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + }, + }, + ), + "", + ), + # SUCCESS: case w/ oneOf property + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + } + }, + { + "properties": { + "api_key": {"type": "string"}, + } + }, + ], + } + }, + }, + authSpecification={ + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", 0], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + }, + }, + ), + "", + ), + # FAIL: Wrong root object index + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + } + }, + { + "properties": { + "api_key": {"type": "string"}, + } + }, + ], + } + }, + }, + authSpecification={ + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", 1], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + }, + }, + ), + "Specified oauth fields are missed from spec schema:", + ), + # SUCCESS: root object index equal to 1 + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "properties": { + "api_key": {"type": "string"}, + } + }, + { + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + } + }, + ], + } + }, + }, + authSpecification={ + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", 1], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + }, + }, + ), + "", + ), + ], +) +def test_validate_oauth_flow(connector_spec, expected_error): + t = _TestSpec() + if expected_error: + with pytest.raises(AssertionError, match=expected_error): + t.test_oauth_flow_parameters(connector_spec) + else: + t.test_oauth_flow_parameters(connector_spec) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index c5c4d9107a02b..8caa65d60a444 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -811,7 +811,7 @@ void testCustomDbtTransformationsFailure() throws Exception { final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform")); final OperatorDbt dbtConfig = new OperatorDbt() .withGitRepoUrl("https://github.com/fishtown-analytics/dbt-learn-demo.git") - .withGitRepoBranch("master") + .withGitRepoBranch("main") .withDockerImage("fishtownanalytics/dbt:0.19.1") .withDbtArguments("debug"); if (!runner.run(JOB_ID, JOB_ATTEMPT, transformationRoot, config, null, dbtConfig)) { diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java index 1c3d309049647..4c6bedec4c831 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.standardtest.source; -import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; @@ -13,6 +13,7 @@ import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; import io.airbyte.config.EnvConfigs; +import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.workers.WorkerConfigs; @@ -65,10 +66,15 @@ protected JsonNode getState() throws IOException { } @Override - protected List getRegexTests() throws IOException { - return Streams.stream(runExecutable(Command.GET_REGEX_TESTS).withArray("tests").elements()) - .map(JsonNode::textValue) - .collect(toList()); + protected void assertFullRefreshMessages(final List allMessages) throws IOException { + final List regexTests = Streams.stream(runExecutable(Command.GET_REGEX_TESTS).withArray("tests").elements()) + .map(JsonNode::textValue).toList(); + final List stringMessages = allMessages.stream().map(Jsons::serialize).toList(); + LOGGER.info("Running " + regexTests.size() + " regex tests..."); + regexTests.forEach(regex -> { + LOGGER.info("Looking for [" + regex + "]"); + assertTrue(stringMessages.stream().anyMatch(line -> line.matches(regex)), "Failed to find regex: " + regex); + }); } @Override diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java index 60772c479d2a4..6d858261fc46c 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java @@ -16,6 +16,7 @@ import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; import io.airbyte.config.StandardCheckConnectionOutput.Status; +import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -105,14 +106,6 @@ public abstract class SourceAcceptanceTest extends AbstractSourceConnectorTest { */ protected abstract JsonNode getState() throws Exception; - /** - * List of regular expressions that should match the output of the test sync. - * - * @return the regular expressions to test - * @throws Exception - thrown when attempting ot access the regexes fails - */ - protected abstract List getRegexTests() throws Exception; - /** * Verify that a spec operation issued to the connector returns a valid spec. */ @@ -147,9 +140,16 @@ public void testCheckConnection() throws Exception { */ @Test public void testDiscover() throws Exception { - // the worker validates that it is a valid catalog, so we do not need to validate again (as long as - // we use the worker, which we will not want to do long term). - assertNotNull(runDiscover(), "Expected discover to produce a catalog"); + final AirbyteCatalog discoverOutput = runDiscover(); + assertNotNull(discoverOutput, "Expected discover to produce a catalog"); + verifyCatalog(discoverOutput); + } + + /** + * Override this method to check the actual catalog. + */ + protected void verifyCatalog(final AirbyteCatalog catalog) throws Exception { + // do nothing by default } /** @@ -160,23 +160,15 @@ public void testDiscover() throws Exception { public void testFullRefreshRead() throws Exception { final ConfiguredAirbyteCatalog catalog = withFullRefreshSyncModes(getConfiguredCatalog()); final List allMessages = runRead(catalog); - final List recordMessages = filterRecords(allMessages); - // the worker validates the message formats, so we just validate the message content - // We don't need to validate message format as long as we use the worker, which we will not want to - // do long term. - assertFalse(recordMessages.isEmpty(), "Expected a full refresh sync to produce records"); - assertRecordMessages(recordMessages); - - final List regexTests = getRegexTests(); - final List stringMessages = allMessages.stream().map(Jsons::serialize).collect(Collectors.toList()); - LOGGER.info("Running " + regexTests.size() + " regex tests..."); - regexTests.forEach(regex -> { - LOGGER.info("Looking for [" + regex + "]"); - assertTrue(stringMessages.stream().anyMatch(line -> line.matches(regex)), "Failed to find regex: " + regex); - }); + + assertFalse(filterRecords(allMessages).isEmpty(), "Expected a full refresh sync to produce records"); + assertFullRefreshMessages(allMessages); } - protected void assertRecordMessages(final List recordMessages) { + /** + * Override this method to perform more specific assertion on the messages. + */ + protected void assertFullRefreshMessages(final List allMessages) throws Exception { // do nothing by default } @@ -287,7 +279,7 @@ public void testEntrypointEnvVar() throws Exception { checkEntrypointEnvVariable(); } - private List filterRecords(final Collection messages) { + protected static List filterRecords(final Collection messages) { return messages.stream() .filter(m -> m.getType() == Type.RECORD) .map(AirbyteMessage::getRecord) diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs index 9588875eb6255..690741787f929 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs @@ -38,7 +38,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", "type": "string", "order": 5 }, diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs index 39d2eb9260325..7eebca640c098 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs @@ -11,9 +11,7 @@ import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; -import java.util.Collections; import java.util.HashMap; -import java.util.List; public class {{pascalCase name}}SourceAcceptanceTest extends SourceAcceptanceTest { @@ -55,11 +53,6 @@ public class {{pascalCase name}}SourceAcceptanceTest extends SourceAcceptanceTes return null; } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index e283bc9062d5e..83c1e535a81a9 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery-denormalized +ENV APPLICATION_VERSION 0.2.6 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java index 4d4f79cc59c80..cdc862ee31abd 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java @@ -27,7 +27,7 @@ protected String getTargetTableName(final String streamName) { } @Override - protected Map getFormatterMap(JsonNode jsonSchema) { + protected Map getFormatterMap(final JsonNode jsonSchema) { return Map.of(UploaderType.STANDARD, new DefaultBigQueryDenormalizedRecordFormatter(jsonSchema, getNamingResolver()), UploaderType.AVRO, new GcsBigQueryDenormalizedRecordFormatter(jsonSchema, getNamingResolver())); } @@ -47,9 +47,7 @@ protected boolean isDefaultAirbyteTmpTableSchema() { public static void main(final String[] args) throws Exception { final Destination destination = new BigQueryDenormalizedDestination(); - LOGGER.info("starting destination: {}", BigQueryDenormalizedDestination.class); new IntegrationRunner(destination).run(args); - LOGGER.info("completed destination: {}", BigQueryDenormalizedDestination.class); } } diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 57decbb9135eb..8fe9d8d415a04 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery +ENV APPLICATION_VERSION 0.6.6 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.5 +LABEL io.airbyte.version=0.6.6 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index e243a59ef9b73..b90fc557dccc1 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -157,7 +157,7 @@ protected Map> getUp for (final ConfiguredAirbyteStream configStream : catalog.getStreams()) { final AirbyteStream stream = configStream.getStream(); final String streamName = stream.getName(); - UploaderConfig uploaderConfig = UploaderConfig + final UploaderConfig uploaderConfig = UploaderConfig .builder() .bigQuery(bigquery) .configStream(configStream) @@ -186,7 +186,7 @@ protected boolean isDefaultAirbyteTmpTableSchema() { return true; } - protected Map getFormatterMap(JsonNode jsonSchema) { + protected Map getFormatterMap(final JsonNode jsonSchema) { return Map.of(UploaderType.STANDARD, new DefaultBigQueryRecordFormatter(jsonSchema, getNamingResolver()), UploaderType.CSV, new GcsCsvBigQueryRecordFormatter(jsonSchema, getNamingResolver()), UploaderType.AVRO, new GcsAvroBigQueryRecordFormatter(jsonSchema, getNamingResolver())); @@ -203,9 +203,7 @@ protected AirbyteMessageConsumer getRecordConsumer(final Map getTypeToDestination() { public static void main(final String[] args) throws Exception { final Destination destination = new SnowflakeDestination(); - LOGGER.info("starting destination: {}", SnowflakeDestination.class); new IntegrationRunner(destination).run(args); - LOGGER.info("completed destination: {}", SnowflakeDestination.class); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index e1db16e984a31..da52ad207e3e3 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -9,6 +9,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.sentry.AirbyteSentry; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; @@ -27,13 +28,15 @@ public SnowflakeInternalStagingDestination() { } @Override - public AirbyteConnectionStatus check(JsonNode config) { - SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); - SnowflakeStagingSqlOperations snowflakeStagingSqlOperations = new SnowflakeStagingSqlOperations(); + public AirbyteConnectionStatus check(final JsonNode config) { + final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); + final SnowflakeStagingSqlOperations snowflakeStagingSqlOperations = new SnowflakeStagingSqlOperations(); try (final JdbcDatabase database = getDatabase(config)) { final String outputSchema = super.getNamingResolver().getIdentifier(config.get("schema").asText()); - attemptSQLCreateAndDropTableOperations(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations); - attemptSQLCreateAndDropStages(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations); + AirbyteSentry.runWithSpan("CreateAndDropTable", + () -> attemptSQLCreateAndDropTableOperations(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations)); + AirbyteSentry.runWithSpan("CreateAndDropStage", + () -> attemptSQLCreateAndDropStages(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations)); return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); } catch (final Exception e) { LOGGER.error("Exception while checking connection: ", e); @@ -43,15 +46,15 @@ public AirbyteConnectionStatus check(JsonNode config) { } } - private static void attemptSQLCreateAndDropStages(String outputSchema, - JdbcDatabase database, - SnowflakeSQLNameTransformer namingResolver, - SnowflakeStagingSqlOperations sqlOperations) + private static void attemptSQLCreateAndDropStages(final String outputSchema, + final JdbcDatabase database, + final SnowflakeSQLNameTransformer namingResolver, + final SnowflakeStagingSqlOperations sqlOperations) throws Exception { // verify we have permissions to create/drop stage final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); - String stageName = namingResolver.getStageName(outputSchema, outputTableName);; + final String stageName = namingResolver.getStageName(outputSchema, outputTableName);; sqlOperations.createStageIfNotExists(database, stageName); sqlOperations.dropStageIfExists(database, stageName); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index 914aee0d1aac1..f6830cfd21bdf 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -20,7 +20,7 @@ "additionalProperties": true, "properties": { "host": { - "description": "Host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com).", + "description": "The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com).", "examples": ["accountname.us-east-2.aws.snowflakecomputing.com"], "type": "string", "title": "Host", @@ -48,7 +48,7 @@ "order": 3 }, "schema": { - "description": "The default Snowflake schema tables are written to if the source does not specify a namespace. Schema name would be transformed to allowed by Snowflake if it not follow Snowflake Naming Conventions https://docs.airbyte.io/integrations/destinations/snowflake#notes-about-snowflake-naming-conventions ", + "description": "The default schema is used as the target schema for all statements issued from the connection that do not explicitly specify a schema name.. Schema name would be transformed to allowed by Snowflake if it not follow Snowflake Naming Conventions https://docs.airbyte.io/integrations/destinations/snowflake#notes-about-snowflake-naming-conventions ", "examples": ["AIRBYTE_SCHEMA"], "type": "string", "title": "Default Schema", @@ -62,17 +62,23 @@ "order": 5 }, "password": { - "description": "Password associated with the username.", + "description": "The password associated with the username.", "type": "string", "airbyte_secret": true, "title": "Password", "order": 6 }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 7 + }, "loading_method": { "type": "object", "title": "Loading Method", - "description": "Loading method used to send data to Snowflake.", - "order": 7, + "description": "The loading method used to send data to Snowflake.", + "order": 8, "oneOf": [ { "title": "[Recommended] Internal Staging", @@ -128,7 +134,7 @@ "title": "S3 Bucket Region", "type": "string", "default": "", - "description": "The region of the S3 staging bucket to use if utilising a copy strategy.", + "description": "The region of the S3 staging bucket which is used when utilising a copy strategy.", "enum": [ "", "us-east-1", diff --git a/airbyte-integrations/connectors/source-bigcommerce/Dockerfile b/airbyte-integrations/connectors/source-bigcommerce/Dockerfile index 95df50bf140fe..6c2015507f23a 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/Dockerfile +++ b/airbyte-integrations/connectors/source-bigcommerce/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-bigcommerce diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json index 5772a8c906927..937e43f367145 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json @@ -10,5 +10,8 @@ }, "pages": { "id": 9000002039398998 + }, + "products": { + "date_modified": "2080-01-10T00:17:08+00:00" } } diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/catalog.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/catalog.json index 46ed0c25d5c39..d99135edad664 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/catalog.json @@ -473,7 +473,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "currency": { "type": ["null", "string"] @@ -671,6 +671,806 @@ } } } + }, + { + "name": "products", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "weight": { + "type": ["null", "number"], + "format": "float" + }, + "width": { + "type": ["null", "number"], + "format": "float" + }, + "depth": { + "type": ["null", "number"], + "format": "float" + }, + "height": { + "type": ["null", "number"], + "format": "float" + }, + "price": { + "type": ["null", "number"], + "format": "float" + }, + "cost_price": { + "type": ["null", "number"], + "format": "float" + }, + "retail_price": { + "type": ["null", "number"], + "format": "float" + }, + "sale_price": { + "type": ["null", "number"], + "format": "float" + }, + "map_price": { + "type": ["null", "number"] + }, + "tax_class_id": { + "type": ["null", "integer"] + }, + "product_tax_code": { + "type": ["null", "string"] + }, + "categories": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "brand_id": { + "type": ["null", "integer"] + }, + "inventory_level": { + "type": ["null", "integer"] + }, + "inventory_warning_level": { + "type": ["null", "integer"] + }, + "inventory_tracking": { + "type": ["null", "string"] + }, + "fixed_cost_shipping_price": { + "type": ["null", "number"], + "format": "float" + }, + "is_free_shipping": { + "type": ["null", "boolean"] + }, + "is_visible": { + "type": ["null", "boolean"] + }, + "is_featured": { + "type": ["null", "boolean"] + }, + "related_products": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "warranty": { + "type": ["null", "string"] + }, + "bin_picking_number": { + "type": ["null", "string"] + }, + "layout_file": { + "type": ["null", "string"] + }, + "upc": { + "type": ["null", "string"] + }, + "search_keywords": { + "type": ["null", "string"] + }, + "availability": { + "type": ["null", "string"] + }, + "availability_description": { + "type": ["null", "string"] + }, + "gift_wrapping_options_type": { + "type": ["null", "string"] + }, + "gift_wrapping_options_list": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sort_order": { + "type": ["null", "integer"] + }, + "condition": { + "type": ["null", "string"] + }, + "is_condition_shown": { + "type": ["null", "boolean"] + }, + "order_quantity_minimum": { + "type": ["null", "integer"] + }, + "order_quantity_maximum": { + "type": ["null", "integer"] + }, + "page_title": { + "type": ["null", "string"] + }, + "meta_keywords": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "meta_description": { + "type": ["null", "string"] + }, + "view_count": { + "type": ["null", "integer"] + }, + "preorder_release_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "preorder_message": { + "type": ["null", "string"] + }, + "is_preorder_only": { + "type": ["null", "boolean"] + }, + "is_price_hidden": { + "type": ["null", "boolean"] + }, + "price_hidden_label": { + "type": ["null", "string"] + }, + "custom_url": { + "type": ["null", "object"], + "title": "customUrl_Full", + "properties": { + "url": { + "type": ["null", "string"] + }, + "is_customized": { + "type": ["null", "boolean"] + } + } + }, + "open_graph_type": { + "type": ["null", "string"] + }, + "open_graph_title": { + "type": ["null", "string"] + }, + "open_graph_description": { + "type": ["null", "string"] + }, + "open_graph_use_meta_description": { + "type": ["null", "boolean"] + }, + "open_graph_use_product_name": { + "type": ["null", "boolean"] + }, + "open_graph_use_image": { + "type": ["null", "boolean"] + }, + "brand_name or brand_id": { + "type": ["null", "string"] + }, + "gtin": { + "type": ["null", "string"] + }, + "mpn": { + "type": ["null", "string"] + }, + "reviews_rating_sum": { + "type": ["null", "integer"] + }, + "reviews_count": { + "type": ["null", "integer"] + }, + "total_sold": { + "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "productCustomField_Put", + "required": ["name", "value"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "bulk_pricing_rules": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "bulkPricingRule_Full", + "required": ["quantity_min", "quantity_max", "type", "amount"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "quantity_min": { + "type": ["null", "integer"] + }, + "quantity_max": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + } + } + } + }, + "images": { + "type": ["null", "array"], + "items": { + "title": "productImage_Full", + "type": ["null", "object"], + "properties": { + "image_file": { + "type": ["null", "string"] + }, + "is_thumbnail": { + "type": ["null", "boolean"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "image_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "url_zoom": { + "type": ["null", "string"] + }, + "url_standard": { + "type": ["null", "string"] + }, + "url_thumbnail": { + "type": ["null", "string"] + }, + "url_tiny": { + "type": ["null", "string"] + }, + "date_modified": { + "format": "date-time", + "type": ["null", "string"] + } + } + } + }, + "videos": { + "type": ["null", "array"], + "items": { + "title": "productVideo_Full", + "type": ["null", "object"], + "properties": { + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "video_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "length": { + "type": ["null", "string"] + } + } + } + }, + "date_created": { + "type": ["null", "string"], + "format": "date-time" + }, + "date_modified": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "base_variant_id": { + "type": ["null", "integer"] + }, + "calculated_price": { + "type": ["null", "number"], + "format": "float" + }, + "options": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "productOption_Base", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "display_name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "config": { + "type": ["null", "object"], + "title": "productOptionConfig_Full", + "properties": { + "default_value": { + "type": ["null", "string"] + }, + "checked_by_default": { + "type": ["null", "boolean"] + }, + "checkbox_label": { + "type": ["null", "string"] + }, + "date_limited": { + "type": ["null", "boolean"] + }, + "date_limit_mode": { + "type": ["null", "string"] + }, + "date_earliest_value": { + "type": ["null", "string"], + "format": "date" + }, + "date_latest_value": { + "type": ["null", "string"], + "format": "date" + }, + "file_types_mode": { + "type": ["null", "string"] + }, + "file_types_supported": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_types_other": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_max_size": { + "type": ["null", "integer"] + }, + "text_characters_limited": { + "type": ["null", "boolean"] + }, + "text_min_length": { + "type": ["null", "integer"] + }, + "text_max_length": { + "type": ["null", "integer"] + }, + "text_lines_limited": { + "type": ["null", "boolean"] + }, + "text_max_lines": { + "type": ["null", "integer"] + }, + "number_limited": { + "type": ["null", "boolean"] + }, + "number_limit_mode": { + "type": ["null", "string"] + }, + "number_lowest_value": { + "type": ["null", "number"] + }, + "number_highest_value": { + "type": ["null", "number"] + }, + "number_integers_only": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_inventory": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_pricing": { + "type": ["null", "boolean"] + }, + "product_list_shipping_calc": { + "type": ["null", "string"] + } + } + }, + "sort_order": { + "type": ["null", "integer"] + }, + "option_values": { + "title": "productOptionOptionValue_Full", + "type": ["null", "object"], + "required": ["label", "sort_order"], + "properties": { + "is_default": { + "type": ["null", "boolean"] + }, + "label": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "value_data": { + "type": ["object", "null"] + }, + "id": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "modifiers": { + "type": ["null", "array"], + "items": { + "title": "productModifier_Full", + "type": ["null", "object"], + "required": ["type", "required"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "required": { + "type": ["null", "boolean"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "config": { + "type": ["null", "object"], + "title": "config_Full", + "properties": { + "default_value": { + "type": ["null", "string"] + }, + "checked_by_default": { + "type": ["null", "boolean"] + }, + "checkbox_label": { + "type": ["null", "string"] + }, + "date_limited": { + "type": ["null", "boolean"] + }, + "date_limit_mode": { + "type": ["null", "string"] + }, + "date_earliest_value": { + "type": ["null", "string"], + "format": "date" + }, + "date_latest_value": { + "type": ["null", "string"], + "format": "date" + }, + "file_types_mode": { + "type": ["null", "string"] + }, + "file_types_supported": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_types_other": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_max_size": { + "type": ["null", "integer"] + }, + "text_characters_limited": { + "type": ["null", "boolean"] + }, + "text_min_length": { + "type": ["null", "integer"] + }, + "text_max_length": { + "type": ["null", "integer"] + }, + "text_lines_limited": { + "type": ["null", "boolean"] + }, + "text_max_lines": { + "type": ["null", "integer"] + }, + "number_limited": { + "type": ["null", "boolean"] + }, + "number_limit_mode": { + "type": ["null", "string"] + }, + "number_lowest_value": { + "type": ["null", "number"] + }, + "number_highest_value": { + "type": ["null", "number"] + }, + "number_integers_only": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_inventory": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_pricing": { + "type": ["null", "boolean"] + }, + "product_list_shipping_calc": { + "type": ["null", "string"] + } + } + }, + "display_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "option_values": { + "type": ["null", "array"], + "items": { + "title": "productModifierOptionValue_Full", + "type": ["null", "object"], + "required": ["label", "sort_order"], + "properties": { + "is_default": { + "type": ["null", "boolean"] + }, + "label": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "value_data": { + "type": ["object", "null"] + }, + "adjusters": { + "type": ["null", "object"], + "title": "adjusters_Full", + "properties": { + "price": { + "type": ["null", "object"], + "title": "adjuster_Full", + "properties": { + "adjuster": { + "type": ["null", "string"] + }, + "adjuster_value": { + "type": ["null", "number"] + } + } + }, + "weight": { + "type": ["null", "object"], + "title": "adjuster_Full", + "properties": { + "adjuster": { + "type": ["null", "string"] + }, + "adjuster_value": { + "type": ["null", "number"] + } + } + }, + "image_url": { + "type": ["null", "string"] + }, + "purchasing_disabled": { + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "string"] + } + } + } + } + }, + "id": { + "type": ["null", "integer"] + }, + "option_id": { + "type": ["null", "integer"] + } + } + } + } + } + } + }, + "option_set_id": { + "type": ["null", "integer"] + }, + "option_set_display": { + "type": ["null", "string"] + }, + "variants": { + "type": ["null", "array"], + "items": { + "title": "productVariant_Full", + "type": ["null", "object"], + "properties": { + "cost_price": { + "type": ["null", "number"], + "format": "double" + }, + "price": { + "type": ["null", "number"], + "format": "double" + }, + "sale_price": { + "type": ["null", "number"], + "format": "double" + }, + "retail_price": { + "type": ["null", "number"], + "format": "double" + }, + "weight": { + "type": ["null", "number"], + "format": "double" + }, + "width": { + "type": ["null", "number"], + "format": "double" + }, + "height": { + "type": ["null", "number"], + "format": "double" + }, + "depth": { + "type": ["null", "number"], + "format": "double" + }, + "is_free_shipping": { + "type": ["null", "boolean"] + }, + "fixed_cost_shipping_price": { + "type": ["null", "number"], + "format": "double" + }, + "purchasing_disabled": { + "type": ["null", "boolean"] + }, + "purchasing_disabled_message": { + "type": ["null", "string"] + }, + "upc": { + "type": ["null", "string"] + }, + "inventory_level": { + "type": ["null", "integer"] + }, + "inventory_warning_level": { + "type": ["null", "integer"] + }, + "bin_picking_number": { + "type": ["null", "string"] + }, + "mpn": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "sku": { + "type": ["null", "string"] + }, + "sku_id": { + "type": ["null", "integer"] + }, + "option_values": { + "type": ["null", "array"], + "items": { + "title": "productVariantOptionValue_Full", + "type": ["null", "object"], + "properties": { + "option_display_name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "option_id": { + "type": ["null", "integer"] + } + } + } + }, + "calculated_price": { + "type": ["null", "number"], + "format": "double" + }, + "calculated_weight": { + "type": "number" + } + } + } + } + } + } } ] } diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json index eed9d2358c3a2..dc0c06c76149b 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json @@ -490,7 +490,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "currency": { "type": ["null", "string"] @@ -703,6 +703,814 @@ "sync_mode": "incremental", "cursor_field": ["id"], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "weight": { + "type": ["null", "number"], + "format": "float" + }, + "width": { + "type": ["null", "number"], + "format": "float" + }, + "depth": { + "type": ["null", "number"], + "format": "float" + }, + "height": { + "type": ["null", "number"], + "format": "float" + }, + "price": { + "type": ["null", "number"], + "format": "float" + }, + "cost_price": { + "type": ["null", "number"], + "format": "float" + }, + "retail_price": { + "type": ["null", "number"], + "format": "float" + }, + "sale_price": { + "type": ["null", "number"], + "format": "float" + }, + "map_price": { + "type": ["null", "number"] + }, + "tax_class_id": { + "type": ["null", "integer"] + }, + "product_tax_code": { + "type": ["null", "string"] + }, + "categories": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "brand_id": { + "type": ["null", "integer"] + }, + "inventory_level": { + "type": ["null", "integer"] + }, + "inventory_warning_level": { + "type": ["null", "integer"] + }, + "inventory_tracking": { + "type": ["null", "string"] + }, + "fixed_cost_shipping_price": { + "type": ["null", "number"], + "format": "float" + }, + "is_free_shipping": { + "type": ["null", "boolean"] + }, + "is_visible": { + "type": ["null", "boolean"] + }, + "is_featured": { + "type": ["null", "boolean"] + }, + "related_products": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "warranty": { + "type": ["null", "string"] + }, + "bin_picking_number": { + "type": ["null", "string"] + }, + "layout_file": { + "type": ["null", "string"] + }, + "upc": { + "type": ["null", "string"] + }, + "search_keywords": { + "type": ["null", "string"] + }, + "availability": { + "type": ["null", "string"] + }, + "availability_description": { + "type": ["null", "string"] + }, + "gift_wrapping_options_type": { + "type": ["null", "string"] + }, + "gift_wrapping_options_list": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sort_order": { + "type": ["null", "integer"] + }, + "condition": { + "type": ["null", "string"] + }, + "is_condition_shown": { + "type": ["null", "boolean"] + }, + "order_quantity_minimum": { + "type": ["null", "integer"] + }, + "order_quantity_maximum": { + "type": ["null", "integer"] + }, + "page_title": { + "type": ["null", "string"] + }, + "meta_keywords": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "meta_description": { + "type": ["null", "string"] + }, + "view_count": { + "type": ["null", "integer"] + }, + "preorder_release_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "preorder_message": { + "type": ["null", "string"] + }, + "is_preorder_only": { + "type": ["null", "boolean"] + }, + "is_price_hidden": { + "type": ["null", "boolean"] + }, + "price_hidden_label": { + "type": ["null", "string"] + }, + "custom_url": { + "type": ["null", "object"], + "title": "customUrl_Full", + "properties": { + "url": { + "type": ["null", "string"] + }, + "is_customized": { + "type": ["null", "boolean"] + } + } + }, + "open_graph_type": { + "type": ["null", "string"] + }, + "open_graph_title": { + "type": ["null", "string"] + }, + "open_graph_description": { + "type": ["null", "string"] + }, + "open_graph_use_meta_description": { + "type": ["null", "boolean"] + }, + "open_graph_use_product_name": { + "type": ["null", "boolean"] + }, + "open_graph_use_image": { + "type": ["null", "boolean"] + }, + "brand_name or brand_id": { + "type": ["null", "string"] + }, + "gtin": { + "type": ["null", "string"] + }, + "mpn": { + "type": ["null", "string"] + }, + "reviews_rating_sum": { + "type": ["null", "integer"] + }, + "reviews_count": { + "type": ["null", "integer"] + }, + "total_sold": { + "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "productCustomField_Put", + "required": ["name", "value"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "bulk_pricing_rules": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "bulkPricingRule_Full", + "required": ["quantity_min", "quantity_max", "type", "amount"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "quantity_min": { + "type": ["null", "integer"] + }, + "quantity_max": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + } + } + } + }, + "images": { + "type": ["null", "array"], + "items": { + "title": "productImage_Full", + "type": ["null", "object"], + "properties": { + "image_file": { + "type": ["null", "string"] + }, + "is_thumbnail": { + "type": ["null", "boolean"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "image_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "url_zoom": { + "type": ["null", "string"] + }, + "url_standard": { + "type": ["null", "string"] + }, + "url_thumbnail": { + "type": ["null", "string"] + }, + "url_tiny": { + "type": ["null", "string"] + }, + "date_modified": { + "format": "date-time", + "type": ["null", "string"] + } + } + } + }, + "videos": { + "type": ["null", "array"], + "items": { + "title": "productVideo_Full", + "type": ["null", "object"], + "properties": { + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "video_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "length": { + "type": ["null", "string"] + } + } + } + }, + "date_created": { + "type": ["null", "string"], + "format": "date-time" + }, + "date_modified": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "base_variant_id": { + "type": ["null", "integer"] + }, + "calculated_price": { + "type": ["null", "number"], + "format": "float" + }, + "options": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "productOption_Base", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "display_name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "config": { + "type": ["null", "object"], + "title": "productOptionConfig_Full", + "properties": { + "default_value": { + "type": ["null", "string"] + }, + "checked_by_default": { + "type": ["null", "boolean"] + }, + "checkbox_label": { + "type": ["null", "string"] + }, + "date_limited": { + "type": ["null", "boolean"] + }, + "date_limit_mode": { + "type": ["null", "string"] + }, + "date_earliest_value": { + "type": ["null", "string"], + "format": "date" + }, + "date_latest_value": { + "type": ["null", "string"], + "format": "date" + }, + "file_types_mode": { + "type": ["null", "string"] + }, + "file_types_supported": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_types_other": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_max_size": { + "type": ["null", "integer"] + }, + "text_characters_limited": { + "type": ["null", "boolean"] + }, + "text_min_length": { + "type": ["null", "integer"] + }, + "text_max_length": { + "type": ["null", "integer"] + }, + "text_lines_limited": { + "type": ["null", "boolean"] + }, + "text_max_lines": { + "type": ["null", "integer"] + }, + "number_limited": { + "type": ["null", "boolean"] + }, + "number_limit_mode": { + "type": ["null", "string"] + }, + "number_lowest_value": { + "type": ["null", "number"] + }, + "number_highest_value": { + "type": ["null", "number"] + }, + "number_integers_only": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_inventory": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_pricing": { + "type": ["null", "boolean"] + }, + "product_list_shipping_calc": { + "type": ["null", "string"] + } + } + }, + "sort_order": { + "type": ["null", "integer"] + }, + "option_values": { + "title": "productOptionOptionValue_Full", + "type": ["null", "object"], + "required": ["label", "sort_order"], + "properties": { + "is_default": { + "type": ["null", "boolean"] + }, + "label": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "value_data": { + "type": ["object", "null"] + }, + "id": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "modifiers": { + "type": ["null", "array"], + "items": { + "title": "productModifier_Full", + "type": ["null", "object"], + "required": ["type", "required"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "required": { + "type": ["null", "boolean"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "config": { + "type": ["null", "object"], + "title": "config_Full", + "properties": { + "default_value": { + "type": ["null", "string"] + }, + "checked_by_default": { + "type": ["null", "boolean"] + }, + "checkbox_label": { + "type": ["null", "string"] + }, + "date_limited": { + "type": ["null", "boolean"] + }, + "date_limit_mode": { + "type": ["null", "string"] + }, + "date_earliest_value": { + "type": ["null", "string"], + "format": "date" + }, + "date_latest_value": { + "type": ["null", "string"], + "format": "date" + }, + "file_types_mode": { + "type": ["null", "string"] + }, + "file_types_supported": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_types_other": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_max_size": { + "type": ["null", "integer"] + }, + "text_characters_limited": { + "type": ["null", "boolean"] + }, + "text_min_length": { + "type": ["null", "integer"] + }, + "text_max_length": { + "type": ["null", "integer"] + }, + "text_lines_limited": { + "type": ["null", "boolean"] + }, + "text_max_lines": { + "type": ["null", "integer"] + }, + "number_limited": { + "type": ["null", "boolean"] + }, + "number_limit_mode": { + "type": ["null", "string"] + }, + "number_lowest_value": { + "type": ["null", "number"] + }, + "number_highest_value": { + "type": ["null", "number"] + }, + "number_integers_only": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_inventory": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_pricing": { + "type": ["null", "boolean"] + }, + "product_list_shipping_calc": { + "type": ["null", "string"] + } + } + }, + "display_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "option_values": { + "type": ["null", "array"], + "items": { + "title": "productModifierOptionValue_Full", + "type": ["null", "object"], + "required": ["label", "sort_order"], + "properties": { + "is_default": { + "type": ["null", "boolean"] + }, + "label": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "value_data": { + "type": ["object", "null"] + }, + "adjusters": { + "type": ["null", "object"], + "title": "adjusters_Full", + "properties": { + "price": { + "type": ["null", "object"], + "title": "adjuster_Full", + "properties": { + "adjuster": { + "type": ["null", "string"] + }, + "adjuster_value": { + "type": ["null", "number"] + } + } + }, + "weight": { + "type": ["null", "object"], + "title": "adjuster_Full", + "properties": { + "adjuster": { + "type": ["null", "string"] + }, + "adjuster_value": { + "type": ["null", "number"] + } + } + }, + "image_url": { + "type": ["null", "string"] + }, + "purchasing_disabled": { + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "string"] + } + } + } + } + }, + "id": { + "type": ["null", "integer"] + }, + "option_id": { + "type": ["null", "integer"] + } + } + } + } + } + } + }, + "option_set_id": { + "type": ["null", "integer"] + }, + "option_set_display": { + "type": ["null", "string"] + }, + "variants": { + "type": ["null", "array"], + "items": { + "title": "productVariant_Full", + "type": ["null", "object"], + "properties": { + "cost_price": { + "type": ["null", "number"], + "format": "double" + }, + "price": { + "type": ["null", "number"], + "format": "double" + }, + "sale_price": { + "type": ["null", "number"], + "format": "double" + }, + "retail_price": { + "type": ["null", "number"], + "format": "double" + }, + "weight": { + "type": ["null", "number"], + "format": "double" + }, + "width": { + "type": ["null", "number"], + "format": "double" + }, + "height": { + "type": ["null", "number"], + "format": "double" + }, + "depth": { + "type": ["null", "number"], + "format": "double" + }, + "is_free_shipping": { + "type": ["null", "boolean"] + }, + "fixed_cost_shipping_price": { + "type": ["null", "number"], + "format": "double" + }, + "purchasing_disabled": { + "type": ["null", "boolean"] + }, + "purchasing_disabled_message": { + "type": ["null", "string"] + }, + "upc": { + "type": ["null", "string"] + }, + "inventory_level": { + "type": ["null", "integer"] + }, + "inventory_warning_level": { + "type": ["null", "integer"] + }, + "bin_picking_number": { + "type": ["null", "string"] + }, + "mpn": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "sku": { + "type": ["null", "string"] + }, + "sku_id": { + "type": ["null", "integer"] + }, + "option_values": { + "type": ["null", "array"], + "items": { + "title": "productVariantOptionValue_Full", + "type": ["null", "object"], + "properties": { + "option_display_name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "option_id": { + "type": ["null", "integer"] + } + } + } + }, + "calculated_price": { + "type": ["null", "number"], + "format": "double" + }, + "calculated_weight": { + "type": "number" + } + } + } + } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["date_modified"] + }, + "sync_mode": "incremental", + "cursor_field": ["date_modified"], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json index ed441688d344f..b15dc2066f36a 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json @@ -10,5 +10,8 @@ }, "pages": { "id": 4 + }, + "products": { + "date_modified": "2022-01-10T00:17:08+00:00" } } diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/products.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/products.json new file mode 100644 index 0000000000000..af01020e9a35d --- /dev/null +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/products.json @@ -0,0 +1,797 @@ +{ + "type": "object", + "title": "Product Response", + "properties": { + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "weight": { + "type": ["null", "number"], + "format": "float" + }, + "width": { + "type": ["null", "number"], + "format": "float" + }, + "depth": { + "type": ["null", "number"], + "format": "float" + }, + "height": { + "type": ["null", "number"], + "format": "float" + }, + "price": { + "type": ["null", "number"], + "format": "float" + }, + "cost_price": { + "type": ["null", "number"], + "format": "float" + }, + "retail_price": { + "type": ["null", "number"], + "format": "float" + }, + "sale_price": { + "type": ["null", "number"], + "format": "float" + }, + "map_price": { + "type": ["null", "number"] + }, + "tax_class_id": { + "type": ["null", "integer"] + }, + "product_tax_code": { + "type": ["null", "string"] + }, + "categories": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "brand_id": { + "type": ["null", "integer"] + }, + "inventory_level": { + "type": ["null", "integer"] + }, + "inventory_warning_level": { + "type": ["null", "integer"] + }, + "inventory_tracking": { + "type": ["null", "string"] + }, + "fixed_cost_shipping_price": { + "type": ["null", "number"], + "format": "float" + }, + "is_free_shipping": { + "type": ["null", "boolean"] + }, + "is_visible": { + "type": ["null", "boolean"] + }, + "is_featured": { + "type": ["null", "boolean"] + }, + "related_products": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "warranty": { + "type": ["null", "string"] + }, + "bin_picking_number": { + "type": ["null", "string"] + }, + "layout_file": { + "type": ["null", "string"] + }, + "upc": { + "type": ["null", "string"] + }, + "search_keywords": { + "type": ["null", "string"] + }, + "availability": { + "type": ["null", "string"] + }, + "availability_description": { + "type": ["null", "string"] + }, + "gift_wrapping_options_type": { + "type": ["null", "string"] + }, + "gift_wrapping_options_list": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sort_order": { + "type": ["null", "integer"] + }, + "condition": { + "type": ["null", "string"] + }, + "is_condition_shown": { + "type": ["null", "boolean"] + }, + "order_quantity_minimum": { + "type": ["null", "integer"] + }, + "order_quantity_maximum": { + "type": ["null", "integer"] + }, + "page_title": { + "type": ["null", "string"] + }, + "meta_keywords": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "meta_description": { + "type": ["null", "string"] + }, + "view_count": { + "type": ["null", "integer"] + }, + "preorder_release_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "preorder_message": { + "type": ["null", "string"] + }, + "is_preorder_only": { + "type": ["null", "boolean"] + }, + "is_price_hidden": { + "type": ["null", "boolean"] + }, + "price_hidden_label": { + "type": ["null", "string"] + }, + "custom_url": { + "type": ["null", "object"], + "title": "customUrl_Full", + "properties": { + "url": { + "type": ["null", "string"] + }, + "is_customized": { + "type": ["null", "boolean"] + } + } + }, + "open_graph_type": { + "type": ["null", "string"] + }, + "open_graph_title": { + "type": ["null", "string"] + }, + "open_graph_description": { + "type": ["null", "string"] + }, + "open_graph_use_meta_description": { + "type": ["null", "boolean"] + }, + "open_graph_use_product_name": { + "type": ["null", "boolean"] + }, + "open_graph_use_image": { + "type": ["null", "boolean"] + }, + "brand_name or brand_id": { + "type": ["null", "string"] + }, + "gtin": { + "type": ["null", "string"] + }, + "mpn": { + "type": ["null", "string"] + }, + "reviews_rating_sum": { + "type": ["null", "integer"] + }, + "reviews_count": { + "type": ["null", "integer"] + }, + "total_sold": { + "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "productCustomField_Put", + "required": ["name", "value"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "bulk_pricing_rules": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "bulkPricingRule_Full", + "required": ["quantity_min", "quantity_max", "type", "amount"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "quantity_min": { + "type": ["null", "integer"] + }, + "quantity_max": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + } + } + } + }, + "images": { + "type": ["null", "array"], + "items": { + "title": "productImage_Full", + "type": ["null", "object"], + "properties": { + "image_file": { + "type": ["null", "string"] + }, + "is_thumbnail": { + "type": ["null", "boolean"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "image_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "url_zoom": { + "type": ["null", "string"] + }, + "url_standard": { + "type": ["null", "string"] + }, + "url_thumbnail": { + "type": ["null", "string"] + }, + "url_tiny": { + "type": ["null", "string"] + }, + "date_modified": { + "format": "date-time", + "type": ["null", "string"] + } + } + } + }, + "videos": { + "type": ["null", "array"], + "items": { + "title": "productVideo_Full", + "type": ["null", "object"], + "properties": { + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "video_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "length": { + "type": ["null", "string"] + } + } + } + }, + "date_created": { + "type": ["null", "string"], + "format": "date-time" + }, + "date_modified": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "base_variant_id": { + "type": ["null", "integer"] + }, + "calculated_price": { + "type": ["null", "number"], + "format": "float" + }, + "options": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "title": "productOption_Base", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "display_name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "config": { + "type": ["null", "object"], + "title": "productOptionConfig_Full", + "properties": { + "default_value": { + "type": ["null", "string"] + }, + "checked_by_default": { + "type": ["null", "boolean"] + }, + "checkbox_label": { + "type": ["null", "string"] + }, + "date_limited": { + "type": ["null", "boolean"] + }, + "date_limit_mode": { + "type": ["null", "string"] + }, + "date_earliest_value": { + "type": ["null", "string"], + "format": "date" + }, + "date_latest_value": { + "type": ["null", "string"], + "format": "date" + }, + "file_types_mode": { + "type": ["null", "string"] + }, + "file_types_supported": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_types_other": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_max_size": { + "type": ["null", "integer"] + }, + "text_characters_limited": { + "type": ["null", "boolean"] + }, + "text_min_length": { + "type": ["null", "integer"] + }, + "text_max_length": { + "type": ["null", "integer"] + }, + "text_lines_limited": { + "type": ["null", "boolean"] + }, + "text_max_lines": { + "type": ["null", "integer"] + }, + "number_limited": { + "type": ["null", "boolean"] + }, + "number_limit_mode": { + "type": ["null", "string"] + }, + "number_lowest_value": { + "type": ["null", "number"] + }, + "number_highest_value": { + "type": ["null", "number"] + }, + "number_integers_only": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_inventory": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_pricing": { + "type": ["null", "boolean"] + }, + "product_list_shipping_calc": { + "type": ["null", "string"] + } + } + }, + "sort_order": { + "type": ["null", "integer"] + }, + "option_values": { + "title": "productOptionOptionValue_Full", + "type": ["null", "object"], + "required": ["label", "sort_order"], + "properties": { + "is_default": { + "type": ["null", "boolean"] + }, + "label": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "value_data": { + "type": ["object", "null"] + }, + "id": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "modifiers": { + "type": ["null", "array"], + "items": { + "title": "productModifier_Full", + "type": ["null", "object"], + "required": ["type", "required"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "required": { + "type": ["null", "boolean"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "config": { + "type": ["null", "object"], + "title": "config_Full", + "properties": { + "default_value": { + "type": ["null", "string"] + }, + "checked_by_default": { + "type": ["null", "boolean"] + }, + "checkbox_label": { + "type": ["null", "string"] + }, + "date_limited": { + "type": ["null", "boolean"] + }, + "date_limit_mode": { + "type": ["null", "string"] + }, + "date_earliest_value": { + "type": ["null", "string"], + "format": "date" + }, + "date_latest_value": { + "type": ["null", "string"], + "format": "date" + }, + "file_types_mode": { + "type": ["null", "string"] + }, + "file_types_supported": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_types_other": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "file_max_size": { + "type": ["null", "integer"] + }, + "text_characters_limited": { + "type": ["null", "boolean"] + }, + "text_min_length": { + "type": ["null", "integer"] + }, + "text_max_length": { + "type": ["null", "integer"] + }, + "text_lines_limited": { + "type": ["null", "boolean"] + }, + "text_max_lines": { + "type": ["null", "integer"] + }, + "number_limited": { + "type": ["null", "boolean"] + }, + "number_limit_mode": { + "type": ["null", "string"] + }, + "number_lowest_value": { + "type": ["null", "number"] + }, + "number_highest_value": { + "type": ["null", "number"] + }, + "number_integers_only": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_inventory": { + "type": ["null", "boolean"] + }, + "product_list_adjusts_pricing": { + "type": ["null", "boolean"] + }, + "product_list_shipping_calc": { + "type": ["null", "string"] + } + } + }, + "display_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "option_values": { + "type": ["null", "array"], + "items": { + "title": "productModifierOptionValue_Full", + "type": ["null", "object"], + "required": ["label", "sort_order"], + "properties": { + "is_default": { + "type": ["null", "boolean"] + }, + "label": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "integer"] + }, + "value_data": { + "type": ["object", "null"] + }, + "adjusters": { + "type": ["null", "object"], + "title": "adjusters_Full", + "properties": { + "price": { + "type": ["null", "object"], + "title": "adjuster_Full", + "properties": { + "adjuster": { + "type": ["null", "string"] + }, + "adjuster_value": { + "type": ["null", "number"] + } + } + }, + "weight": { + "type": ["null", "object"], + "title": "adjuster_Full", + "properties": { + "adjuster": { + "type": ["null", "string"] + }, + "adjuster_value": { + "type": ["null", "number"] + } + } + }, + "image_url": { + "type": ["null", "string"] + }, + "purchasing_disabled": { + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "string"] + } + } + } + } + }, + "id": { + "type": ["null", "integer"] + }, + "option_id": { + "type": ["null", "integer"] + } + } + } + } + } + } + }, + "option_set_id": { + "type": ["null", "integer"] + }, + "option_set_display": { + "type": ["null", "string"] + }, + "variants": { + "type": ["null", "array"], + "items": { + "title": "productVariant_Full", + "type": ["null", "object"], + "properties": { + "cost_price": { + "type": ["null", "number"], + "format": "double" + }, + "price": { + "type": ["null", "number"], + "format": "double" + }, + "sale_price": { + "type": ["null", "number"], + "format": "double" + }, + "retail_price": { + "type": ["null", "number"], + "format": "double" + }, + "weight": { + "type": ["null", "number"], + "format": "double" + }, + "width": { + "type": ["null", "number"], + "format": "double" + }, + "height": { + "type": ["null", "number"], + "format": "double" + }, + "depth": { + "type": ["null", "number"], + "format": "double" + }, + "is_free_shipping": { + "type": ["null", "boolean"] + }, + "fixed_cost_shipping_price": { + "type": ["null", "number"], + "format": "double" + }, + "purchasing_disabled": { + "type": ["null", "boolean"] + }, + "purchasing_disabled_message": { + "type": ["null", "string"] + }, + "upc": { + "type": ["null", "string"] + }, + "inventory_level": { + "type": ["null", "integer"] + }, + "inventory_warning_level": { + "type": ["null", "integer"] + }, + "bin_picking_number": { + "type": ["null", "string"] + }, + "mpn": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "sku": { + "type": ["null", "string"] + }, + "sku_id": { + "type": ["null", "integer"] + }, + "option_values": { + "type": ["null", "array"], + "items": { + "title": "productVariantOptionValue_Full", + "type": ["null", "object"], + "properties": { + "option_display_name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "option_id": { + "type": ["null", "integer"] + } + } + } + }, + "calculated_price": { + "type": ["null", "number"], + "format": "double" + }, + "calculated_weight": { + "type": "number" + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/transactions.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/transactions.json index c6ab8612b2551..c2731c2cc44da 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/transactions.json @@ -9,7 +9,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "currency": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py index 38437f97bd2a0..8e67b4f00f749 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py @@ -5,14 +5,16 @@ from abc import ABC from email.utils import parsedate_tz -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple +import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer class BigcommerceStream(HttpStream, ABC): @@ -26,12 +28,34 @@ class BigcommerceStream(HttpStream, ABC): filter_field = "date_modified:min" data = "data" + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + def __init__(self, start_date: str, store_hash: str, access_token: str, **kwargs): super().__init__(**kwargs) self.start_date = start_date self.store_hash = store_hash self.access_token = access_token + @transformer.registerCustomTransform + def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + """ + This functions tries to handle the various date-time formats BigCommerce API returns and normalize the values to isoformat. + """ + if "format" in field_schema and field_schema["format"] == "date-time": + if not original_value: # Some dates are empty strings: "". + return None + transformed_value = None + supported_formats = ["YYYY-MM-DD", "YYYY-MM-DDTHH:mm:ssZZ", "YYYY-MM-DDTHH:mm:ss[Z]", "ddd, D MMM YYYY HH:mm:ss ZZ"] + for format in supported_formats: + try: + transformed_value = str(pendulum.from_format(original_value, format)) # str() returns isoformat + except ValueError: + continue + if not transformed_value: + raise ValueError(f"Unsupported date-time format for {original_value}") + return transformed_value + return original_value + @property def url_base(self) -> str: return f"https://api.bigcommerce.com/stores/{self.store_hash}/{self.api_version}/" @@ -53,6 +77,8 @@ def request_params( params.update({"sort": self.order_field}) if next_page_token: params.update(**next_page_token) + else: + params[self.filter_field] = self.start_date return params def request_headers( @@ -87,6 +113,8 @@ def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token # If there is a next page token then we should only send pagination-related parameters. if stream_state: params[self.filter_field] = stream_state.get(self.cursor_field) + else: + params[self.filter_field] = self.start_date return params def filter_records_newer_than_state(self, stream_state: Mapping[str, Any] = None, records_slice: Mapping[str, Any] = None) -> Iterable: @@ -117,6 +145,15 @@ def path(self, **kwargs) -> str: return f"{self.data_field}" +class Products(IncrementalBigcommerceStream): + data_field = "products" + # Override `order_field` bacause Products API do not acept `asc` value + order_field = "date_modified" + + def path(self, **kwargs) -> str: + return f"catalog/{self.data_field}" + + class Orders(IncrementalBigcommerceStream): data_field = "orders" api_version = "v2" @@ -230,4 +267,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Pages(**args), Orders(**args), Transactions(**args), + Products(**args), ] diff --git a/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceAcceptanceTest.java b/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceAcceptanceTest.java index a3e16338f6e46..3932cfd1108a9 100644 --- a/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceAcceptanceTest.java @@ -21,9 +21,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; -import java.util.Collections; import java.util.HashMap; -import java.util.List; public class BigQuerySourceAcceptanceTest extends SourceAcceptanceTest { @@ -95,11 +93,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { Field.of("name", JsonSchemaPrimitive.STRING)); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java index deea69f3cb8ec..a84dfd3dc32eb 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java @@ -24,9 +24,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.testcontainers.containers.ClickHouseContainer; public abstract class AbstractSshClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { @@ -83,11 +81,6 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { startTestContainers(); diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index a5c54c91ef676..4126ae7026402 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -23,9 +23,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.testcontainers.containers.ClickHouseContainer; public class ClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { @@ -79,11 +77,6 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine"); diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java index df11b7a677d4f..58830c69d70a4 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java @@ -22,9 +22,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.jooq.SQLDialect; public class CockroachDbEncryptSourceAcceptanceTest extends SourceAcceptanceTest { @@ -116,11 +114,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java index 6f08829a65244..08f3706017ae8 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java @@ -21,9 +21,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.jooq.SQLDialect; import org.testcontainers.containers.CockroachContainer; @@ -118,11 +116,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java index eeac2e6814599..bb59cbb9fb408 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java @@ -25,9 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.Db2Container; @@ -91,20 +89,15 @@ protected JsonNode getState() { } @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - - @Override - protected void setupEnvironment(TestDestinationEnv environment) throws Exception { + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { db = new Db2Container("ibmcom/db2:11.5.5.0").withCommand().acceptLicense() .withExposedPorts(50000); db.start(); - var certificate = getCertificate(); + final var certificate = getCertificate(); try { convertAndImportCertificate(certificate); - } catch (IOException | InterruptedException e) { + } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } @@ -121,7 +114,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception .build())) .build()); - String jdbcUrl = String.format("jdbc:db2://%s:%s/%s", + final String jdbcUrl = String.format("jdbc:db2://%s:%s/%s", config.get("host").asText(), db.getMappedPort(50000), config.get("db").asText()) + SSL_CONFIG; @@ -154,7 +147,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { new File("certificate.pem").delete(); new File("certificate.der").delete(); new File(KEY_STORE_FILE_PATH).delete(); @@ -186,9 +179,9 @@ private String getCertificate() throws IOException, InterruptedException { return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout(); } - private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException { - Runtime run = Runtime.getRuntime(); - try (PrintWriter out = new PrintWriter("certificate.pem")) { + private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { + final Runtime run = Runtime.getRuntime(); + try (final PrintWriter out = new PrintWriter("certificate.pem")) { out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); @@ -198,8 +191,8 @@ private static void convertAndImportCertificate(String certificate) throws IOExc run); } - private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException { - Process pr = run.exec(cmd); + private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { + final Process pr = run.exec(cmd); if (!pr.waitFor(30, TimeUnit.SECONDS)) { pr.destroy(); throw new RuntimeException("Timeout while executing: " + cmd); diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java index 2800f90204290..8e88eb7fb7140 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java @@ -22,9 +22,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.testcontainers.containers.Db2Container; public class Db2SourceAcceptanceTest extends SourceAcceptanceTest { @@ -81,11 +79,6 @@ protected JsonNode getState() throws Exception { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() throws Exception { - return Collections.emptyList(); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { db = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense(); diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java index bbcb8735be1bf..fb30f88011978 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java @@ -25,9 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.Db2Container; @@ -89,20 +87,15 @@ protected JsonNode getState() { } @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - - @Override - protected void setupEnvironment(TestDestinationEnv environment) throws Exception { + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { db = new Db2Container("ibmcom/db2:11.5.5.0").withCommand().acceptLicense() .withExposedPorts(50000); db.start(); - var certificate = getCertificate(); + final var certificate = getCertificate(); try { convertAndImportCertificate(certificate); - } catch (IOException | InterruptedException e) { + } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } @@ -119,7 +112,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception .build())) .build()); - String jdbcUrl = String.format("jdbc:db2://%s:%s/%s", + final String jdbcUrl = String.format("jdbc:db2://%s:%s/%s", config.get("host").asText(), db.getMappedPort(50000), config.get("db").asText()) + ":sslConnection=true;sslTrustStoreLocation=" + KEY_STORE_FILE_PATH + @@ -153,7 +146,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { new File("certificate.pem").delete(); new File("certificate.der").delete(); new File(KEY_STORE_FILE_PATH).delete(); @@ -180,9 +173,9 @@ private String getCertificate() throws IOException, InterruptedException { return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout(); } - private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException { - Runtime run = Runtime.getRuntime(); - try (PrintWriter out = new PrintWriter("certificate.pem")) { + private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { + final Runtime run = Runtime.getRuntime(); + try (final PrintWriter out = new PrintWriter("certificate.pem")) { out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); @@ -192,8 +185,8 @@ private static void convertAndImportCertificate(String certificate) throws IOExc run); } - private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException { - Process pr = run.exec(cmd); + private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { + final Process pr = run.exec(cmd); if (!pr.waitFor(30, TimeUnit.SECONDS)) { pr.destroy(); throw new RuntimeException("Timeout while executing: " + cmd); diff --git a/airbyte-integrations/connectors/source-delighted/Dockerfile b/airbyte-integrations/connectors/source-delighted/Dockerfile index 41c613ad0ff75..c098a13136594 100644 --- a/airbyte-integrations/connectors/source-delighted/Dockerfile +++ b/airbyte-integrations/connectors/source-delighted/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-delighted diff --git a/airbyte-integrations/connectors/source-delighted/setup.py b/airbyte-integrations/connectors/source-delighted/setup.py index e5f41ecef635a..096214a03e443 100644 --- a/airbyte-integrations/connectors/source-delighted/setup.py +++ b/airbyte-integrations/connectors/source-delighted/setup.py @@ -12,6 +12,7 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "source-acceptance-test", + "responses~=0.13.3", ] setup( diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 80e738728afaa..796c1362410a8 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -18,7 +18,6 @@ # Basic full refresh stream class DelightedStream(HttpStream, ABC): - url_base = "https://api.delighted.com/v1/" # Page size @@ -41,10 +40,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: + params = {"per_page": self.limit, "since": self.since} if next_page_token: - params = {"per_page": self.limit, **next_page_token} - else: - params = {"per_page": self.limit, "since": self.since} + params.update(**next_page_token) return params def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: @@ -52,7 +50,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalDelightedStream(DelightedStream, ABC): - # Getting page size as 'limit' from parrent class + # Getting page size as 'limit' from parent class @property def limit(self): return super().limit @@ -73,8 +71,17 @@ def request_params(self, stream_state=None, **kwargs): params["since"] = stream_state.get(self.cursor_field) return params + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(response=response, stream_state=stream_state, **kwargs): + if self.cursor_field not in stream_state or record[self.cursor_field] > stream_state[self.cursor_field]: + yield record + class People(IncrementalDelightedStream): + """ + API docs: https://app.delighted.com/docs/api/listing-people + """ + def path(self, **kwargs) -> str: return "people.json" @@ -86,6 +93,10 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, class Unsubscribes(IncrementalDelightedStream): + """ + API docs: https://app.delighted.com/docs/api/listing-unsubscribed-people + """ + cursor_field = "unsubscribed_at" primary_key = "person_id" @@ -94,6 +105,10 @@ def path(self, **kwargs) -> str: class Bounces(IncrementalDelightedStream): + """ + API docs: https://app.delighted.com/docs/api/listing-bounced-people + """ + cursor_field = "bounced_at" primary_key = "person_id" @@ -102,6 +117,10 @@ def path(self, **kwargs) -> str: class SurveyResponses(IncrementalDelightedStream): + """ + API docs: https://app.delighted.com/docs/api/listing-survey-responses + """ + cursor_field = "updated_at" def path(self, **kwargs) -> str: @@ -110,8 +129,13 @@ def path(self, **kwargs) -> str: def request_params(self, stream_state=None, **kwargs): stream_state = stream_state or {} params = super().request_params(stream_state=stream_state, **kwargs) + + if "since" in params: + params["updated_since"] = params.pop("since") + if stream_state: params["updated_since"] = stream_state.get(self.cursor_field) + return params diff --git a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py index e1814314fc3b0..b6eddc0ecae5a 100644 --- a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py @@ -2,6 +2,82 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import pytest +import responses +from airbyte_cdk.models import SyncMode +from source_delighted.source import Bounces, People, SourceDelighted, SurveyResponses, Unsubscribes + + +@pytest.fixture(scope="module") +def test_config(): + return { + "api_key": "test_api_key", + "since": "1641289584", + } + + +@pytest.fixture(scope="module") +def state(): + return { + "bounces": {"bounced_at": 1641455286}, + "people": {"created_at": 1641455285}, + "survey_responses": {"updated_at": 1641289816}, + "unsubscribes": {"unsubscribed_at": 1641289584}, + } + + +BOUNCES_RESPONSE = """ +[ + {"person_id": "1046789984", "email": "foo_test204@airbyte.io", "name": "Foo Test204", "bounced_at": 1641455286}, + {"person_id": "1046789989", "email": "foo_test205@airbyte.io", "name": "Foo Test205", "bounced_at": 1641455286} +] +""" + + +PEOPLE_RESPONSE = """ +[ + {"id": "1046789989", "name": "Foo Test205", "email": "foo_test205@airbyte.io", "created_at": 1641455285, "last_sent_at": 1641455285, "last_responded_at": null, "next_survey_scheduled_at": null} +] +""" + + +SURVEY_RESPONSES_RESPONSE = """ +[ + {"id": "210554887", "person": "1042205953", "survey_type": "nps", "score": 0, "comment": "Test Comment202", "permalink": "https://app.delighted.com/r/0q7QEdWzosv5G5c3w9gakivDwEIM5Hq0", "created_at": 1641289816, "updated_at": 1641289816, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, + {"id": "210554885", "person": "1042205947", "survey_type": "nps", "score": 5, "comment": "Test Comment201", "permalink": "https://app.delighted.com/r/GhWWrBT2wayswOc0AfT7fxpM3UwSpitN", "created_at": 1641289816, "updated_at": 1641289816, "person_properties": null, "notes": [], "tags": [], "additional_answers": []} +] +""" + + +UNSUBSCRIBES_RESPONSE = """ +[ + {"person_id": "1040826319", "email": "foo_test64@airbyte.io", "name": "Foo Test64", "unsubscribed_at": 1641289584} +] +""" + + +@pytest.mark.parametrize( + ("stream_class", "url", "response_body"), + [ + (Bounces, "https://api.delighted.com/v1/bounces.json", BOUNCES_RESPONSE), + (People, "https://api.delighted.com/v1/people.json", PEOPLE_RESPONSE), + (SurveyResponses, "https://api.delighted.com/v1/survey_responses.json", SURVEY_RESPONSES_RESPONSE), + (Unsubscribes, "https://api.delighted.com/v1/unsubscribes.json", UNSUBSCRIBES_RESPONSE), + ], +) +@responses.activate +def test_not_output_records_where_cursor_field_equals_state(state, test_config, stream_class, url, response_body): + responses.add( + responses.GET, + url, + body=response_body, + status=200, + ) + + stream = stream_class(test_config["since"], authenticator=SourceDelighted()._get_authenticator(config=test_config)) + records = [r for r in stream.read_records(SyncMode.incremental, stream_state=state[stream.name])] + assert not records + def test_example_method(): assert True diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile index a230cc6117334..d5f623b0cd0b6 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION source-e2e-test-cloud +ENV APPLICATION_VERSION 1.0.1 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-e2e-test-cloud diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/README.md b/airbyte-integrations/connectors/source-e2e-test-cloud/README.md index 43a68d7eb4750..a64de4a140723 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/README.md +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/README.md @@ -1,6 +1,6 @@ # End-to-End Testing Source Cloud Variant -This is the Cloud variant of the [E2E Test Source](https://docs.airbyte.io/integrations/sources/e2e-test). It only allows the "continuous feed" mode with finite number of record messages. +This is the Cloud variant of the [E2E Test Source](https://docs.airbyte.io/integrations/sources/e2e-test). It only allows the "continuous feed" mode a finite number of record messages. The two legacy modes ("infinite feed" and "exception after n") are excluded from cloud because 1) the catalog is not customized under those modes, and 2) the connector should not emit infinite records, which may result in high cost accidentally. ## Local development diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java b/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java index 7e554049c7d7c..a41114cef82ec 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.integrations.source.e2e_test.TestingSources.TestingSourceType; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; @@ -22,14 +23,13 @@ import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.ThreadLocalRandom; /** * This acceptance test is mostly the same as {@code ContinuousFeedSourceAcceptanceTest}. The only - * difference is the image name. + * difference is the image name. TODO: find a way to share classes from integrationTest. */ public class CloudTestingSourcesAcceptanceTest extends SourceAcceptanceTest { @@ -115,12 +115,9 @@ protected JsonNode getState() { } @Override - protected List getRegexTests() { - return Collections.emptyList(); - } + protected void assertFullRefreshMessages(final List allMessages) { + final List recordMessages = filterRecords(allMessages); - @Override - protected void assertRecordMessages(final List recordMessages) { int index = 0; // the first N messages are from stream 1 while (index < MAX_MESSAGES) { diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile index 6ba25c8c6f25f..ddde252871e12 100644 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION source-e2e-test +ENV APPLICATION_VERSION=1.0.1 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-e2e-test diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java index d8d314e500c32..dc58536560d42 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java @@ -18,11 +18,12 @@ import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -91,8 +92,9 @@ static AirbyteCatalog parseMockCatalog(final JsonNode config) throws JsonValidat throw new JsonValidationException("Input stream schemas are invalid: %s" + streamSchemasText); } - final List streams = new LinkedList<>(); - for (final Map.Entry entry : MoreIterators.toList(streamSchemas.get().fields())) { + final List> streamEntries = MoreIterators.toList(streamSchemas.get().fields()); + final List streams = new ArrayList<>(streamEntries.size()); + for (final Map.Entry entry : streamEntries) { final String streamName = entry.getKey(); final JsonNode streamSchema = Jsons.clone(entry.getValue()); processSchema(streamSchema); @@ -121,9 +123,12 @@ private static void checkSchema(final String streamName, final JsonNode streamSc /** * Patch the schema so that 1) it allows no additional properties, and 2) all fields are required. - * This is necessary because the mock Json object generation library may add extra properties, or - * omit non-required fields. TODO (liren): patch the library so we don't need to patch the schema - * here. + * This is necessary because 1) the mock Json object generation library may add extra properties + * with pure random names which look ugly and garbled. 2) We cannot precise customize the library on + * how many non-required fields to include even with the nonRequiredPropertyChance setting in the + * config. To avoid emitting lots of empty objects, all fields are marked as required. TODO (liren): + * update the library so we don't need to patch the schema here. Issue: + * https://github.com/airbytehq/airbyte/issues/9772 */ private static void processSchema(final JsonNode schema) { if (schema.has("type") && schema.get("type").asText().equals("object")) { diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java index 5f5d7b7be50c7..63597145bbdba 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java @@ -60,11 +60,11 @@ public AutoCloseableIterator read(final JsonNode jsonConfig, fin for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { final AtomicLong emittedMessages = new AtomicLong(0); final Optional messageIntervalMs = feedConfig.getMessageIntervalMs(); - final ThreadLocal random = ThreadLocal.withInitial(() -> new Random(feedConfig.getSeed())); final SchemaStore schemaStore = new SchemaStore(true); final Schema schema = schemaStore.loadSchemaJson(Jsons.serialize(stream.getStream().getJsonSchema())); - final Generator generator = new Generator(ContinuousFeedConstants.MOCK_JSON_CONFIG, schemaStore, random.get()); + final Random random = new Random(feedConfig.getSeed()); + final Generator generator = new Generator(ContinuousFeedConstants.MOCK_JSON_CONFIG, schemaStore, random); final Iterator streamIterator = new AbstractIterator<>() { diff --git a/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java index 929fa941488ac..67f6d88470b17 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.integrations.source.e2e_test.TestingSources.TestingSourceType; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; @@ -22,7 +23,6 @@ import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.ThreadLocalRandom; @@ -111,12 +111,9 @@ protected JsonNode getState() { } @Override - protected List getRegexTests() { - return Collections.emptyList(); - } + protected void assertFullRefreshMessages(final List allMessages) { + final List recordMessages = filterRecords(allMessages); - @Override - protected void assertRecordMessages(final List recordMessages) { int index = 0; // the first N messages are from stream 1 while (index < MAX_MESSAGES) { diff --git a/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSourceAcceptanceTest.java index 5504d86c78520..3681cd373b8d5 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSourceAcceptanceTest.java @@ -14,9 +14,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; -import java.util.List; public class LegacyInfiniteFeedSourceAcceptanceTest extends SourceAcceptanceTest { @@ -60,9 +58,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - } diff --git a/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfigTest.java b/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfigTest.java index 06ffc24fa1a8c..2fdf6d20d61b2 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfigTest.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfigTest.java @@ -16,7 +16,6 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.validation.json.JsonValidationException; import java.util.Optional; -import java.util.Random; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; @@ -32,17 +31,16 @@ class ContinuousFeedConfigTest { private static final Logger LOGGER = LoggerFactory.getLogger(ContinuousFeedConfigTest.class); private static final ObjectMapper MAPPER = MoreMappers.initMapper(); - private static final Random RANDOM = new Random(); @Test public void testParseSeed() { - final long seed = RANDOM.nextLong(); + final long seed = 1029L; assertEquals(seed, ContinuousFeedConfig.parseSeed(Jsons.deserialize(String.format("{ \"seed\": %d }", seed)))); } @Test public void testParseMaxMessages() { - final long maxMessages = RANDOM.nextLong(); + final long maxMessages = 68373L; assertEquals(maxMessages, ContinuousFeedConfig.parseMaxMessages(Jsons.deserialize(String.format("{ \"max_messages\": %d }", maxMessages)))); } @@ -62,12 +60,10 @@ public Stream provideArguments(final ExtensionContext conte Jsons.deserialize(MoreResources.readResource("parse_mock_catalog_test_cases.json")); return MoreIterators.toList(testCases.elements()).stream().map(testCase -> { final JsonNode sourceConfig = MAPPER.createObjectNode().set("mock_catalog", testCase.get("mockCatalog")); - final boolean invalidSchema = testCase.has("invalidSchema") && testCase.get("invalidSchema").asBoolean(); - final AirbyteCatalog expectedCatalog = invalidSchema ? null : Jsons.object(testCase.get("expectedCatalog"), AirbyteCatalog.class); + final AirbyteCatalog expectedCatalog = Jsons.object(testCase.get("expectedCatalog"), AirbyteCatalog.class); return Arguments.of( testCase.get("testCase").asText(), sourceConfig, - invalidSchema, expectedCatalog); }); } @@ -78,10 +74,9 @@ public Stream provideArguments(final ExtensionContext conte @ArgumentsSource(ContinuousFeedConfigTestCaseProvider.class) public void testParseMockCatalog(final String testCaseName, final JsonNode mockConfig, - final boolean invalidSchema, final AirbyteCatalog expectedCatalog) throws Exception { - if (invalidSchema) { + if (expectedCatalog == null) { assertThrows(JsonValidationException.class, () -> ContinuousFeedConfig.parseMockCatalog(mockConfig)); } else { final AirbyteCatalog actualCatalog = ContinuousFeedConfig.parseMockCatalog(mockConfig); diff --git a/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json b/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json index cdb86a1688b88..1bf65b74122ad 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json +++ b/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json @@ -34,7 +34,7 @@ "stream_name": "my_stream", "stream_schema": "[123, 456]" }, - "invalidSchema": true + "expectedCatalog": null }, { "testCase": "single stream with invalid schema", @@ -43,7 +43,7 @@ "stream_name": "my_stream", "stream_schema": "{ \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"invalid_type\" }, \"field2\": { \"type\": \"number\" } } }" }, - "invalidSchema": true + "expectedCatalog": null }, { "testCase": "multi stream", @@ -91,7 +91,7 @@ "type": "MULTI_STREAM", "stream_schemas": "{ \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" }, \"field2\": { \"type\": \"number\" } } }" }, - "invalidSchema": true + "expectedCatalog": null }, { "testCase": "multi stream with invalid schema", @@ -99,6 +99,6 @@ "type": "MULTI_STREAM", "stream_schemas": "{ \"stream1\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" }, \"field2\": [\"invalid field spec\"] } }, \"stream2\": { \"type\": \"object\", \"properties\": { \"column1\": { \"type\": \"string\" } } } }" }, - "invalidSchema": true + "expectedCatalog": null } ] diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index 9823d188cb033..e3ca3ccb2cbd3 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.14 +LABEL io.airbyte.version=0.2.15 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json index f64a7ea03c410..8e4cd1e554159 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json @@ -340,7 +340,21 @@ "type": ["null", "string"] }, "auto_merge": { - "type": ["null", "boolean"] + "type": ["null", "object"], + "properties": { + "enabled_by": { + "$ref": "user.json" + }, + "commit_title": { + "type": ["null", "string"] + }, + "merge_method": { + "type": ["null", "string"] + }, + "commit_message": { + "type": ["null", "string"] + } + } }, "draft": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json index 58d02c8655326..61786a73b2a3e 100644 --- a/airbyte-integrations/connectors/source-github/source_github/spec.json +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -54,9 +54,13 @@ }, "repository": { "type": "string", - "examples": ["airbytehq/airbyte", "airbytehq/*"], + "examples": [ + "airbytehq/airbyte airbytehq/another-repo", + "airbytehq/*", + "airbytehq/airbyte" + ], "title": "GitHub Repositories", - "description": "Space-delimited list of GitHub repositories/organizations, e.g. `airbytehq/airbyte` for single repository and `airbytehq/*` for get all repositories from organization" + "description": "Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories." }, "start_date": { "type": "string", @@ -68,7 +72,7 @@ "branch": { "type": "string", "title": "Branch", - "examples": ["airbytehq/airbyte/master"], + "examples": ["airbytehq/airbyte/master airbytehq/airbyte/my-branch"], "description": "Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled." }, "page_size_for_large_streams": { diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile index d1215de0e6410..fb470d3c967e7 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.15 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py index e6cb85dcc3a2d..11d9c84b6b515 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "PyJWT", "cryptography"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "PyJWT", "cryptography", "requests"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index 535e551c5dd26..15ada1ca0c518 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -4,6 +4,7 @@ import json +import logging import pkgutil import time from abc import ABC @@ -18,6 +19,13 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +DATA_IS_NOT_GOLDEN_MSG = "Google Analytics data is not golden. Future requests may return different data." + +RESULT_IS_SAMPLED_MSG = ( + "Google Analytics data is sampled. Consider using a smaller window_in_days parameter. " + "For more info check https://developers.google.com/analytics/devguides/reporting/core/v4/basics#sampling" +) + class GoogleAnalyticsV4TypesList(HttpStream): """ @@ -32,14 +40,14 @@ class GoogleAnalyticsV4TypesList(HttpStream): # Column id completely match for v3 and v4. url_base = "https://www.googleapis.com/analytics/v3/metadata/ga/columns" - def path(self, **kwargs) -> str: + def path(self, **kwargs: Any) -> str: return "" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """Abstractmethod HTTPStream CDK dependency""" return None - def parse_response(self, response: requests.Response, **kwargs) -> Tuple[dict, dict]: + def parse_response(self, response: requests.Response, **kwargs: Any) -> Tuple[dict, dict]: """ Returns a map of (dimensions, metrics) hashes, example: ({"ga:userType": "STRING", "ga:sessionCount": "STRING"}, {"ga:pageviewsPerSession": "FLOAT", "ga:sessions": "INTEGER"}) @@ -88,16 +96,18 @@ class GoogleAnalyticsV4Stream(HttpStream, ABC): map_type = dict(INTEGER="integer", FLOAT="number", PERCENT="number", TIME="number") - def __init__(self, config: Dict): + def __init__(self, config: MutableMapping): super().__init__(authenticator=config["authenticator"]) self.start_date = config["start_date"] - self.window_in_days = config.get("window_in_days", 90) + self.window_in_days: int = config.get("window_in_days", 1) self.view_id = config["view_id"] self.metrics = config["metrics"] self.dimensions = config["dimensions"] self._config = config self.dimensions_ref, self.metrics_ref = GoogleAnalyticsV4TypesList().read_records(sync_mode=None) + self._raise_on_http_errors: bool = True + @property def state_checkpoint_interval(self) -> int: return self.window_in_days @@ -115,7 +125,7 @@ def to_datetime_str(date: datetime) -> str: def to_iso_datetime_str(date: str) -> str: return datetime.strptime(date, "%Y%m%d").strftime("%Y-%m-%d") - def path(self, **kwargs) -> str: + def path(self, **kwargs: Any) -> str: # need add './' for correct urllib.parse.urljoin work due to path contains ':' return "./reports:batchGet" @@ -131,15 +141,17 @@ def should_retry(self, response: requests.Response) -> bool: if response.status_code == 400: self.logger.info(f"{response.json()['error']['message']}") - self.raise_on_http_errors = False + self._raise_on_http_errors = False - return super().should_retry(response) + result: bool = HttpStream.should_retry(self, response) + return result + @property def raise_on_http_errors(self) -> bool: - return True + return self._raise_on_http_errors def request_body_json( - self, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + self, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs: Any ) -> Optional[Mapping]: metrics = [{"expression": metric} for metric in self.metrics] @@ -166,7 +178,7 @@ def get_json_schema(self) -> Mapping[str, Any]: Override get_json_schema CDK method to retrieve the schema information for GoogleAnalyticsV4 Object dynamically. """ - schema = { + schema: Dict[str, Any] = { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "additionalProperties": False, @@ -181,7 +193,7 @@ def get_json_schema(self) -> Mapping[str, Any]: data_format = self.lookup_data_format(dimension) dimension = dimension.replace("ga:", "ga_") - dimension_data = {"type": [data_type]} + dimension_data: Dict[str, Any] = {"type": [data_type]} if data_format: dimension_data["format"] = data_format schema["properties"][dimension] = dimension_data @@ -193,14 +205,14 @@ def get_json_schema(self) -> Mapping[str, Any]: metric = metric.replace("ga:", "ga_") # metrics are allowed to also have null values - metric_data = {"type": ["null", data_type]} + metric_data: Dict[str, Any] = {"type": ["null", data_type]} if data_format: metric_data["format"] = data_format schema["properties"][metric] = metric_data return schema - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs: Any) -> Iterable[Optional[Mapping[str, Any]]]: """ Override default stream_slices CDK method to provide date_slices as page chunks for data fetch. Returns list of dict, example: [{ @@ -233,7 +245,8 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite start_date = end_date_slice.add(days=1) return date_slices - def get_data(self, data): + # TODO: the method has to be updated for more logical and obvious + def get_data(self, data): # type: ignore[no-untyped-def] for data_field in self.data_fields: if data and isinstance(data, dict): data = data.get(data_field, []) @@ -241,7 +254,7 @@ def get_data(self, data): return [] return data - def lookup_data_type(self, field_type, attribute): + def lookup_data_type(self, field_type: str, attribute: str) -> str: """ Get the data type of a metric or a dimension """ @@ -274,17 +287,14 @@ def lookup_data_type(self, field_type, attribute): attr_type = None self.logger.error(f"Unsuported GA {field_type}: {attribute}") - data_type = self.map_type.get(attr_type, "string") - - return data_type + return self.map_type.get(attr_type, "string") @staticmethod def lookup_data_format(attribute: str) -> Union[str, None]: if attribute == "ga:date": return "date" - return - def convert_to_type(self, header, value, data_type): + def convert_to_type(self, header: str, value: Any, data_type: str) -> Any: if data_type == "integer": return int(value) if data_type == "number": @@ -293,7 +303,7 @@ def convert_to_type(self, header, value, data_type): return self.to_iso_datetime_str(value) return value - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + def parse_response(self, response: requests.Response, **kwargs: Any) -> Iterable[Mapping]: """ Default response: @@ -375,6 +385,8 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp dimension_headers = column_header.get("dimensions", []) metric_headers = column_header.get("metricHeader", {}).get("metricHeaderEntries", []) + self.check_for_sampled_result(report.get("data", {})) + for row in self.get_data(report): record = {} dimensions = row.get("dimensions", []) @@ -398,6 +410,12 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield record + def check_for_sampled_result(self, data: Mapping) -> None: + if not data.get("isDataGolden", True): + self.logger.warning(DATA_IS_NOT_GOLDEN_MSG) + if data.get("samplesReadCounts", False): + self.logger.warning(RESULT_IS_SAMPLED_MSG) + class GoogleAnalyticsV4IncrementalObjectsBase(GoogleAnalyticsV4Stream): cursor_field = "ga_date" @@ -415,7 +433,7 @@ class GoogleAnalyticsServiceOauth2Authenticator(Oauth2Authenticator): https://oauth2.googleapis.com/token?grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=signed_JWT """ - def __init__(self, config): + def __init__(self, config: Mapping): self.credentials_json = json.loads(config["credentials_json"]) self.client_email = self.credentials_json["client_email"] self.scope = "https://www.googleapis.com/auth/analytics.readonly" @@ -449,7 +467,7 @@ def refresh_access_token(self) -> Tuple[str, int]: else: return response_json["access_token"], response_json["expires_in"] - def get_refresh_request_params(self) -> Mapping[str, any]: + def get_refresh_request_params(self) -> Mapping[str, Any]: """ Sign the JWT with RSA-256 using the private key found in service account JSON file. """ @@ -483,7 +501,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, """For test reading pagination is not required""" return None - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs: Any) -> Iterable[Optional[Mapping[str, Any]]]: """ Override this method to fetch records from start_date up to now for testing case """ @@ -496,7 +514,7 @@ class SourceGoogleAnalyticsV4(AbstractSource): """Google Analytics lets you analyze data about customer engagement with your website or application.""" @staticmethod - def get_authenticator(config): + def get_authenticator(config: Mapping) -> Oauth2Authenticator: # backwards compatibility, credentials_json used to be in the top level of the connector if config.get("credentials_json"): return GoogleAnalyticsServiceOauth2Authenticator(config) @@ -514,7 +532,7 @@ def get_authenticator(config): scopes=["https://www.googleapis.com/auth/analytics.readonly"], ) - def check_connection(self, logger, config) -> Tuple[bool, any]: + def check_connection(self, logger: logging.Logger, config: MutableMapping) -> Tuple[bool, Any]: # declare additional variables authenticator = self.get_authenticator(config) @@ -547,7 +565,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: else: return False, f"{error_msg}" - def streams(self, config: Mapping[str, Any]) -> List[Stream]: + def streams(self, config: MutableMapping[str, Any]) -> List[Stream]: streams: List[GoogleAnalyticsV4Stream] = [] authenticator = self.get_authenticator(config) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json index 2aeb998bc2ad8..ba9bbf9227ea9 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json @@ -20,6 +20,13 @@ "description": "The date in the format YYYY-MM-DD. Any data before this date will not be replicated.", "examples": ["2020-06-01"] }, + "window_in_days": { + "type": "integer", + "title": "Window in days", + "description": "The amount of days for each data-chunk beginning from start_date. Bigger the value - faster the fetch. (Min=1, as for a Day; Max=364, as for a Year).", + "examples": [30, 60, 90, 120, 200, 364], + "default": 1 + }, "custom_reports": { "order": 3, "type": "string", @@ -30,6 +37,7 @@ "order": 0, "type": "object", "title": "Credentials", + "description": "Credentials for the service", "oneOf": [ { "title": "Authenticate via Google (Oauth)", diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/configured_catalog.json new file mode 100644 index 0000000000000..a69585300322a --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/configured_catalog.json @@ -0,0 +1,48 @@ +{ + "streams": [ + { + "stream": { + "name": "website_overview", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pages", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "monthly_active_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "devices", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/response_is_data_golden_false.json b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/response_is_data_golden_false.json new file mode 100644 index 0000000000000..4e2e641ac3f28 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/response_is_data_golden_false.json @@ -0,0 +1,47 @@ +{ + "reports": [ + { + "columnHeader": { + "dimensions": ["ga: date"], + "metricHeader": { + "metricHeaderEntries": [ + { + "name": "ga: 14dayUsers", + "type": "INTEGER" + } + ] + } + }, + "data": { + "rows": [ + { + "dimensions": ["20201027"], + "metrics": [ + { + "values": ["1"] + } + ] + } + ], + "isDataGolden": false, + "totals": [ + { + "values": ["158"] + } + ], + "rowCount": 134, + "minimums": [ + { + "values": ["0"] + } + ], + "maximums": [ + { + "values": ["3"] + } + ] + }, + "nextPageToken": "1" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/response_with_sampling.json b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/response_with_sampling.json new file mode 100644 index 0000000000000..b116b5f012621 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/response_with_sampling.json @@ -0,0 +1,48 @@ +{ + "reports": [ + { + "columnHeader": { + "dimensions": ["ga: date"], + "metricHeader": { + "metricHeaderEntries": [ + { + "name": "ga: 14dayUsers", + "type": "INTEGER" + } + ] + } + }, + "data": { + "rows": [ + { + "dimensions": ["20201027"], + "metrics": [ + { + "values": ["1"] + } + ] + } + ], + "samplesReadCounts": ["499630", "499630"], + "samplingSpaceSizes": ["15328013", "15328013"], + "totals": [ + { + "values": ["158"] + } + ], + "rowCount": 134, + "minimums": [ + { + "values": ["0"] + } + ], + "maximums": [ + { + "values": ["3"] + } + ] + }, + "nextPageToken": "1" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py index 084e0970d0e85..6af81b43f58a6 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py @@ -3,15 +3,19 @@ # import json +import logging from pathlib import Path from unittest.mock import MagicMock, patch from urllib.parse import unquote import pendulum import pytest +from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.sources.streams.http.auth import NoAuth from freezegun import freeze_time from source_google_analytics_v4.source import ( + DATA_IS_NOT_GOLDEN_MSG, + RESULT_IS_SAMPLED_MSG, GoogleAnalyticsV4IncrementalObjectsBase, GoogleAnalyticsV4Stream, GoogleAnalyticsV4TypesList, @@ -25,24 +29,34 @@ def read_file(file_name): return file -expected_metrics_dimensions_type_map = ({"ga:users": "INTEGER", "ga:newUsers": "INTEGER"}, {"ga:date": "STRING", "ga:country": "STRING"}) +expected_metrics_dimensions_type_map = ( + {"ga:users": "INTEGER", "ga:newUsers": "INTEGER"}, + {"ga:date": "STRING", "ga:country": "STRING"}, +) @pytest.fixture def mock_metrics_dimensions_type_list_link(requests_mock): requests_mock.get( - "https://www.googleapis.com/analytics/v3/metadata/ga/columns", json=json.loads(read_file("metrics_dimensions_type_list.json")) + "https://www.googleapis.com/analytics/v3/metadata/ga/columns", + json=json.loads(read_file("metrics_dimensions_type_list.json")), ) @pytest.fixture def mock_auth_call(requests_mock): - yield requests_mock.post("https://oauth2.googleapis.com/token", json={"access_token": "", "expires_in": 0}) + yield requests_mock.post( + "https://oauth2.googleapis.com/token", + json={"access_token": "", "expires_in": 0}, + ) @pytest.fixture def mock_auth_check_connection(requests_mock): - yield requests_mock.post("https://analyticsreporting.googleapis.com/v4/reports:batchGet", json={"data": {"test": "value"}}) + yield requests_mock.post( + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + json={"data": {"test": "value"}}, + ) @pytest.fixture @@ -58,7 +72,8 @@ def mock_unknown_metrics_or_dimensions_error(requests_mock): def mock_api_returns_no_records(requests_mock): """API returns empty data for given date based slice""" yield requests_mock.post( - "https://analyticsreporting.googleapis.com/v4/reports:batchGet", json=json.loads(read_file("empty_response.json")) + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + json=json.loads(read_file("empty_response.json")), ) @@ -66,7 +81,26 @@ def mock_api_returns_no_records(requests_mock): def mock_api_returns_valid_records(requests_mock): """API returns valid data for given date based slice""" yield requests_mock.post( - "https://analyticsreporting.googleapis.com/v4/reports:batchGet", json=json.loads(read_file("response_with_records.json")) + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + json=json.loads(read_file("response_with_records.json")), + ) + + +@pytest.fixture +def mock_api_returns_sampled_results(requests_mock): + """API returns valid data for given date based slice""" + yield requests_mock.post( + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + json=json.loads(read_file("response_with_sampling.json")), + ) + + +@pytest.fixture +def mock_api_returns_is_data_golden_false(requests_mock): + """API returns valid data for given date based slice""" + yield requests_mock.post( + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + json=json.loads(read_file("response_is_data_golden_false.json")), ) @@ -76,6 +110,9 @@ def test_config(): test_config["authenticator"] = NoAuth() test_config["metrics"] = [] test_config["dimensions"] = [] + test_config["credentials"] = { + "type": "Service", + } return test_config @@ -100,15 +137,60 @@ def get_metrics_dimensions_mapping(): def test_lookup_metrics_dimensions_data_type(test_config, metrics_dimensions_mapping, mock_metrics_dimensions_type_list_link): field_type, attribute, expected = metrics_dimensions_mapping g = GoogleAnalyticsV4Stream(config=test_config) - test = g.lookup_data_type(field_type, attribute) - assert test == expected +def test_data_is_not_golden_is_logged_as_warning( + mock_api_returns_is_data_golden_false, + test_config, + mock_metrics_dimensions_type_list_link, + mock_auth_call, + caplog, +): + source = SourceGoogleAnalyticsV4() + del test_config["custom_reports"] + catalog = ConfiguredAirbyteCatalog.parse_obj(json.loads(read_file("./configured_catalog.json"))) + list(source.read(logging.getLogger(), test_config, catalog)) + assert DATA_IS_NOT_GOLDEN_MSG in caplog.text + + +def test_sampled_result_is_logged_as_warning( + mock_api_returns_sampled_results, + test_config, + mock_metrics_dimensions_type_list_link, + mock_auth_call, + caplog, +): + source = SourceGoogleAnalyticsV4() + del test_config["custom_reports"] + catalog = ConfiguredAirbyteCatalog.parse_obj(json.loads(read_file("./configured_catalog.json"))) + list(source.read(logging.getLogger(), test_config, catalog)) + assert RESULT_IS_SAMPLED_MSG in caplog.text + + +def test_no_regressions_for_result_is_sampled_and_data_is_golden_warnings( + mock_api_returns_valid_records, + test_config, + mock_metrics_dimensions_type_list_link, + mock_auth_call, + caplog, +): + source = SourceGoogleAnalyticsV4() + del test_config["custom_reports"] + catalog = ConfiguredAirbyteCatalog.parse_obj(json.loads(read_file("./configured_catalog.json"))) + list(source.read(logging.getLogger(), test_config, catalog)) + assert RESULT_IS_SAMPLED_MSG not in caplog.text + assert DATA_IS_NOT_GOLDEN_MSG not in caplog.text + + @patch("source_google_analytics_v4.source.jwt") def test_check_connection_fails_jwt( - jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call, mock_api_returns_no_records + jwt_encode_mock, + mocker, + mock_metrics_dimensions_type_list_link, + mock_auth_call, + mock_api_returns_no_records, ): """ check_connection fails because of the API returns no records, @@ -133,7 +215,11 @@ def test_check_connection_fails_jwt( @patch("source_google_analytics_v4.source.jwt") def test_check_connection_success_jwt( - jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call, mock_api_returns_valid_records + jwt_encode_mock, + mocker, + mock_metrics_dimensions_type_list_link, + mock_auth_call, + mock_api_returns_valid_records, ): """ check_connection succeeds because of the API returns valid records for the latest date based slice, @@ -156,7 +242,11 @@ def test_check_connection_success_jwt( @patch("source_google_analytics_v4.source.jwt") def test_check_connection_fails_oauth( - jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call, mock_api_returns_no_records + jwt_encode_mock, + mocker, + mock_metrics_dimensions_type_list_link, + mock_auth_call, + mock_api_returns_no_records, ): """ check_connection fails because of the API returns no records, @@ -187,7 +277,11 @@ def test_check_connection_fails_oauth( @patch("source_google_analytics_v4.source.jwt") def test_check_connection_success_oauth( - jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call, mock_api_returns_valid_records + jwt_encode_mock, + mocker, + mock_metrics_dimensions_type_list_link, + mock_auth_call, + mock_api_returns_valid_records, ): """ check_connection succeeds because of the API returns valid records for the latest date based slice, diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 5684beb2d00b6..e5ba6153b1f11 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.35 +LABEL io.airbyte.version=0.1.37 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md index 82a4c9f4a8453..9b849f2a78f85 100644 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ b/airbyte-integrations/connectors/source-hubspot/README.md @@ -47,10 +47,10 @@ and place them into `secrets/config.json`. ### Locally running the connector ``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` ## Testing diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index b7aef4adb1f50..ce27eb7333646 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -17,15 +17,17 @@ tests: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" + timeout_seconds: 600 configured_catalog_path: "sample_files/full_refresh_catalog.json" - empty_streams: ["workflows", "form_submissions", "ticket_pipelines"] + empty_streams: ["workflows", "form_submissions", "ticket_pipelines", "property_history"] - config_path: "secrets/config_oauth.json" + timeout_seconds: 600 configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json" # The `campaigns` stream is empty in this case, because we use a catalog with # incremental streams: subscription_changes and email_events (it takes a long time to read) # and therefore the start date is set at 2021-10-10 for `config_oauth.json`, # but the campaign was created on 2021-01-11 - empty_streams: ["campaigns", "workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines"] + empty_streams: ["campaigns", "workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines", "property_history"] incremental: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/configured_catalog.json" @@ -34,6 +36,7 @@ tests: subscription_changes: ["timestamp"] email_events: ["timestamp"] contact_lists: ["timestamp"] + property_history: ["timestamp"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/full_refresh_catalog.json" @@ -68,6 +71,8 @@ tests: "tickets": ["properties", "hs_time_in_2"] "tickets": ["properties", "hs_time_in_3"] "tickets": ["properties", "hs_time_in_4"] + "property_history": ["property", "hs_time_in_lead"] + "property_history": ["property", "hs_time_in_subscriber"] - config_path: "secrets/config_oauth.json" configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json" ignored_fields: @@ -101,3 +106,5 @@ tests: "tickets": ["properties", "hs_time_in_2"] "tickets": ["properties", "hs_time_in_3"] "tickets": ["properties", "hs_time_in_4"] + "property_history": ["property", "hs_time_in_lead"] + "property_history": ["property", "hs_time_in_subscriber"] diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json index 8fb486ae13c4e..3492fad076232 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json @@ -120,6 +120,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "form_submissions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "line_items", @@ -165,6 +174,17 @@ "cursor_field": ["updatedAt"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": ["timestamp"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["timestamp"], + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "quotes", diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json index 673f27f562138..08402c4ee1645 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json @@ -84,6 +84,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "form_submissions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "forms", @@ -141,6 +150,17 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": ["timestamp"] + }, + "default_cursor_field": ["timestamp"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "subscription_changes", diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json index 76b06a1c7dd24..aeccc52d02f79 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json @@ -117,6 +117,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "quotes", diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json index 470938cb24c4f..eac403b2d9bee 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json @@ -1,6 +1,7 @@ { "start_date": "2020-01-01T00:00:00Z", "credentials": { + "credentials_title": "API Key Credentials", "api_key": "demo" } } diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config_oauth.json b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config_oauth.json new file mode 100644 index 0000000000000..f3b7f165c557b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config_oauth.json @@ -0,0 +1,9 @@ +{ + "start_date": "2021-10-01T00:00:00Z", + "credentials": { + "credentials_title": "OAuth Credentials", + "client_id": "123456789_client_id_hubspot", + "client_secret": "123456789_client_secret_hubspot", + "refresh_token": "123456789_some_refresh_token" + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 0a366001472a6..d6d5d0ccc6067 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from functools import lru_cache, partial from http import HTTPStatus -from typing import Any, Callable, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union import backoff import pendulum as pendulum @@ -348,10 +348,14 @@ def _read(self, getter: Callable, params: MutableMapping[str, Any] = None) -> It if not next_page_token: break - def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: + def read(self, getter: Callable, params: Mapping[str, Any] = None, filter_old_records: bool = True) -> Iterator: default_params = {self.limit_field: self.limit} params = {**default_params, **params} if params else {**default_params} - yield from self._filter_old_records(self._read(getter, params)) + generator = self._read(getter, params) + if filter_old_records: + generator = self._filter_old_records(generator) + + yield from generator def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iterator: if isinstance(response, Mapping): @@ -879,6 +883,10 @@ class FormSubmissionStream(Stream): limit = 50 updated_at_field = "updatedAt" + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.forms = FormStream(**kwargs) + def _transform(self, records: Iterable) -> Iterable: for record in super()._transform(records): keys = record.keys() @@ -891,10 +899,14 @@ def _transform(self, records: Iterable) -> Iterable: yield record def list_records(self, fields) -> Iterable: - for form in self.read(getter=partial(self._api.get, url="/marketing/v3/forms")): - for submission in self.read(getter=partial(self._api.get, url=f"{self.url}/{form['id']}")): - submission["formId"] = form["id"] - yield submission + seen = set() + # To get submissions for all forms date filtering has to be disabled + for form in self.forms.read(getter=partial(self.forms._api.get, url=self.forms.url), filter_old_records=False): + if form["id"] not in seen: + seen.add(form["id"]) + for submission in self.read(getter=partial(self._api.get, url=f"{self.url}/{form['id']}")): + submission["formId"] = form["id"] + yield submission class MarketingEmailStream(Stream): @@ -919,6 +931,50 @@ class OwnerStream(Stream): created_at_field = "createdAt" +class PropertyHistoryStream(IncrementalStream): + """Contacts Endpoint, API v1 + Is used to get all Contacts and the history of their respective + Properties. Whenever a property is changed it is added here. + Docs: https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts + """ + + more_key = "has-more" + url = "/contacts/v1/lists/recently_updated/contacts/recent" + updated_at_field = "timestamp" + created_at_field = "timestamp" + data_field = "contacts" + page_field = "vid-offset" + page_filter = "vidOffset" + limit = 100 + + def list(self, fields) -> Iterable: + properties = self._api.get("/properties/v2/contact/properties") + properties_list = [single_property["name"] for single_property in properties] + params = {"propertyMode": "value_and_history", "property": properties_list} + yield from self.read(partial(self._api.get, url=self.url), params) + + def _transform(self, records: Iterable) -> Iterable: + for record in records: + properties = record.get("properties") + vid = record.get("vid") + value_dict: Dict + for key, value_dict in properties.items(): + versions = value_dict.get("versions") + if key == "lastmodifieddate": + # Skipping the lastmodifieddate since it only returns the value + # when one field of a contact was changed no matter which + # field was changed. It therefore creates overhead, since for + # every changed property there will be the date it was changed in itself + # and a change in the lastmodifieddate field. + continue + if versions: + for version in versions: + version["timestamp"] = self._field_to_datetime(version["timestamp"]).to_datetime_string() + version["property"] = key + version["vid"] = vid + yield version + + class SubscriptionChangeStream(IncrementalStream): """Subscriptions timeline for a portal, API v1 Docs: https://legacydocs.hubspot.com/docs/methods/email/get_subscriptions_timeline diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py index 6dcffff072bb7..c77dccc8c43c7 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py @@ -23,6 +23,7 @@ FormSubmissionStream, MarketingEmailStream, OwnerStream, + PropertyHistoryStream, SubscriptionChangeStream, TicketPipelineStream, WorkflowStream, @@ -58,6 +59,7 @@ def __init__(self, start_date, credentials, **kwargs): "marketing_emails": MarketingEmailStream(**common_params), "owners": OwnerStream(**common_params), "products": CRMObjectIncrementalStream(entity="product", **common_params), + "property_history": PropertyHistoryStream(**common_params), "subscription_changes": SubscriptionChangeStream(**common_params), "tickets": CRMObjectIncrementalStream(entity="ticket", **common_params), "ticket_pipelines": TicketPipelineStream(**common_params), diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attribute_values.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json similarity index 52% rename from airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attribute_values.json rename to airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json index c1473efaf6814..13ae1f80322bd 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attribute_values.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json @@ -1,36 +1,34 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "$schema": "http://json-schema.org/draft-07/schema", + "type": ["null", "object"], "properties": { - "hidden_value_domain_whitelist": { + "value": { "type": ["null", "string"] }, - "label": { + "source-type": { "type": ["null", "string"] }, - "name": { + "source-id": { "type": ["null", "string"] }, - "rank": { - "type": ["null", "integer"] - }, - "source": { + "source-label": { "type": ["null", "string"] }, - "user_attribute_id": { + "updated-by-user-id": { "type": ["null", "integer"] }, - "user_can_edit": { - "type": ["null", "boolean"] + "timestamp": { + "type": ["null", "string"], + "format": "date-time" }, - "user_id": { - "type": ["null", "integer"] + "selected": { + "type": ["null", "boolean"] }, - "value": { + "property": { "type": ["null", "string"] }, - "value_is_hidden": { - "type": ["null", "boolean"] + "vid": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java index 2b46801b8f175..75680be008204 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java @@ -18,9 +18,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.sql.SQLException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import org.testcontainers.containers.PostgreSQLContainer; /** @@ -97,9 +95,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() throws Exception { - return new ArrayList<>(); - } - } diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index e2ba1c704ef2e..a288fa5300b69 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; @@ -69,8 +70,10 @@ // 4. Then implement the abstract methods documented below. public abstract class JdbcSourceAcceptanceTest { - public static String SCHEMA_NAME = "jdbc_integration_test1"; - public static String SCHEMA_NAME2 = "jdbc_integration_test2"; + // schema name must be randomized for each test run, + // otherwise parallel runs can interfere with each other + public static String SCHEMA_NAME = Strings.addRandomSuffix("jdbc_integration_test1", "_", 5).toLowerCase(); + public static String SCHEMA_NAME2 = Strings.addRandomSuffix("jdbc_integration_test2", "_", 5).toLowerCase(); public static Set TEST_SCHEMAS = ImmutableSet.of(SCHEMA_NAME, SCHEMA_NAME2); public static String TABLE_NAME = "id_and_name"; diff --git a/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java index b3dff865ea88a..f5731dc399397 100644 --- a/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java @@ -22,7 +22,6 @@ import io.airbyte.protocol.models.SyncMode; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import org.apache.kafka.clients.admin.AdminClient; @@ -125,9 +124,4 @@ protected JsonNode getState() throws Exception { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() throws Exception { - return Collections.emptyList(); - } - } diff --git a/airbyte-integrations/connectors/source-looker/Dockerfile b/airbyte-integrations/connectors/source-looker/Dockerfile index 87853aa790b43..4e30e99f66386 100644 --- a/airbyte-integrations/connectors/source-looker/Dockerfile +++ b/airbyte-integrations/connectors/source-looker/Dockerfile @@ -33,5 +33,5 @@ COPY source_looker ./source_looker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.6 +LABEL io.airbyte.version=0.2.7 LABEL io.airbyte.name=airbyte/source-looker diff --git a/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml index 848cac5714f46..31daa5289820f 100644 --- a/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml @@ -16,16 +16,24 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [ - "homepages", - "looks", - "run_looks", - "scheduled_plans", - "user_attribute_group_values", - "user_login_lockouts", - "user_sessions", + "scheduled_plans", + "user_attribute_group_values", + "user_login_lockouts", + "user_sessions", ] full_refresh: + # test streams except "run_looks" - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_without_dynamic_streams.json" - ignored_fields: - "datagroups": ["properties", "trigger_check_at"] + configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + datagroups: [ "properties", "trigger_check_at" ] + looks: [ "properties", "last_accessed_at" ] + # test the stream "run_looks" separately + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_run_looks_catalog.json" + + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + diff --git a/airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh index e4d8b1cef8961..c51577d10690c 100644 --- a/airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-looker/integration_tests/abnormal_state.json new file mode 100755 index 0000000000000..4fc01b57eda45 --- /dev/null +++ b/airbyte-integrations/connectors/source-looker/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "query_history": { + "history_created_time": "2050-01-01T00:00:00Z" + } +} diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py index 056971f954502..e7663f3a634c5 100644 --- a/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py @@ -3,14 +3,14 @@ # +from typing import Iterable + import pytest pytest_plugins = ("source_acceptance_test.plugin",) @pytest.fixture(scope="session", autouse=True) -def connector_setup(): +def connector_setup() -> Iterable: """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog.json index f70573d9fdfaf..b11bac063778c 100644 --- a/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog.json @@ -4,371 +4,662 @@ "stream": { "name": "color_collections", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "connections", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "content_metadata", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "content_metadata_access", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "dashboards", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" }, { "stream": { "name": "dashboard_elements", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "dashboard_filters", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "dashboard_layouts", + "name": "dashboard_layout_components", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "dashboards", + "name": "dashboard_layouts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "datagroups", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "folders", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "folder_ancestors", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" }, { "stream": { "name": "git_branches", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "groups", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "homepage_items", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "homepage_sections", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" }, { "stream": { "name": "homepages", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "integration_hubs", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "integrations", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "legacy_features", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "lookml_dashboards", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "lookml_models", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "looks", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "model_sets", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "permission_sets", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "permissions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "project_files", + "name": "primary_homepage_sections", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "projects", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "query_history", + "name": "project_files", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "role_groups", + "name": "query_history", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "history_created_time" + ], + "source_defined_primary_key": [ + [ + "query_id" + ], + [ + "history_created_time" + ] + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "append" }, { "stream": { "name": "roles", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "run_looks", + "name": "role_groups", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "scheduled_plans", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "spaces", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "user_attribute_group_values", + "name": "space_ancestors", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "user_attribute_values", + "name": "user_attributes", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { - "name": "user_attributes", + "name": "user_attribute_group_values", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "user_attribute_values", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "user_login_lockouts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "user_sessions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "users", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "versions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" }, { "stream": { "name": "workspaces", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog_without_dynamic_streams.json b/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog_without_dynamic_streams.json deleted file mode 100644 index b04d7d0193ba1..0000000000000 --- a/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog_without_dynamic_streams.json +++ /dev/null @@ -1,364 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "color_collections", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "connections", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "content_metadata", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "content_metadata_access", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "dashboard_elements", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "dashboard_filters", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "dashboard_layouts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "dashboards", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "datagroups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "folders", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "git_branches", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "homepages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "integration_hubs", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "integrations", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "lookml_dashboards", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "lookml_models", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "looks", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "model_sets", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "permission_sets", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "permissions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "project_files", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "projects", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "role_groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "roles", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "run_looks", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "scheduled_plans", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "spaces", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "user_attribute_group_values", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "user_attribute_values", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "user_attributes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "user_login_lockouts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "user_sessions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "versions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "workspaces", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/configured_run_looks_catalog.json b/airbyte-integrations/connectors/source-looker/integration_tests/configured_run_looks_catalog.json new file mode 100644 index 0000000000000..6bdc61884beb3 --- /dev/null +++ b/airbyte-integrations/connectors/source-looker/integration_tests/configured_run_looks_catalog.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "run_looks", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-looker/integration_tests/sample_config.json deleted file mode 100644 index 6fb15afc1b088..0000000000000 --- a/airbyte-integrations/connectors/source-looker/integration_tests/sample_config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "domain": "", - "client_id": "", - "client_secret": "" -} diff --git a/airbyte-integrations/connectors/source-looker/setup.py b/airbyte-integrations/connectors/source-looker/setup.py index 70757024058e1..19c56ae9127ee 100644 --- a/airbyte-integrations/connectors/source-looker/setup.py +++ b/airbyte-integrations/connectors/source-looker/setup.py @@ -6,7 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ + "types-requests", "airbyte-cdk~=0.1", + "prance~=0.21.8", + "openapi_spec_validator~=0.3.1", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-looker/source_looker/client.py b/airbyte-integrations/connectors/source-looker/source_looker/client.py deleted file mode 100644 index 1549f8e80be56..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/client.py +++ /dev/null @@ -1,419 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from typing import Generator, List, Tuple - -import backoff -import requests -from airbyte_cdk.models import AirbyteStream -from airbyte_cdk.sources.deprecated.client import BaseClient -from requests.exceptions import ConnectionError -from requests.structures import CaseInsensitiveDict - - -class Client(BaseClient): - API_VERSION = "3.1" - - def __init__(self, domain: str, client_id: str, client_secret: str, run_look_ids: list = []): - """ - Note that we dynamically generate schemas for the stream__run_looks - function because the fields returned depend on the user's look(s) - (entered during configuration). See get_run_look_json_schema(). - """ - self.BASE_URL = f"https://{domain}/api/{self.API_VERSION}" - self._client_id = client_id - self._client_secret = client_secret - self._token, self._connect_error = self.get_token() - self._headers = { - "Authorization": f"token {self._token}", - "Content-Type": "application/json", - "Accept": "application/json", - } - - # Maps Looker types to JSON Schema types for run_look JSON schema - self._field_type_mapping = { - "string": "string", - "date_date": "datetime", - "date_raw": "datetime", - "date": "datetime", - "date_week": "datetime", - "date_day_of_week": "string", - "date_day_of_week_index": "integer", - "date_month": "string", - "date_month_num": "integer", - "date_month_name": "string", - "date_day_of_month": "integer", - "date_fiscal_month_num": "integer", - "date_quarter": "string", - "date_quarter_of_year": "string", - "date_fiscal_quarter": "string", - "date_fiscal_quarter_of_year": "string", - "date_year": "integer", - "date_day_of_year": "integer", - "date_week_of_year": "integer", - "date_fiscal_year": "integer", - "date_time_of_day": "string", - "date_hour": "string", - "date_hour_of_day": "integer", - "date_minute": "datetime", - "date_second": "datetime", - "date_millisecond": "datetime", - "date_microsecond": "datetime", - "number": "number", - "int": "integer", - "list": "array", - "yesno": "boolean", - } - - # Helpers for the self.stream__run_looks function - self._run_look_explore_fields = {} - self._run_looks, self._run_looks_connect_error = self.get_run_look_info(run_look_ids) - - self._dashboard_ids = [] - self._project_ids = [] - self._role_ids = [] - self._user_attribute_ids = [] - self._user_ids = [] - self._context_metadata_mapping = {"dashboards": [], "folders": [], "homepages": [], "looks": [], "spaces": []} - super().__init__() - - @property - def streams(self) -> Generator[AirbyteStream, None, None]: - """ - Uses the default streams except for the run_look endpoint, where we have - to generate its JSON Schema on the fly for the given look - """ - - streams = super().streams - for stream in streams: - if len(self._run_looks) > 0 and stream.name == "run_looks": - stream.json_schema = self._get_run_look_json_schema() - yield stream - - def get_token(self): - headers = CaseInsensitiveDict() - headers["Content-Type"] = "application/x-www-form-urlencoded" - try: - resp = requests.post( - url=f"{self.BASE_URL}/login", headers=headers, data=f"client_id={self._client_id}&client_secret={self._client_secret}" - ) - if resp.status_code != 200: - return None, "Unable to connect to the Looker API. Please check your credentials." - return resp.json()["access_token"], None - except ConnectionError as error: - return None, str(error) - - def get_run_look_info(self, run_look_ids): - """ - Checks that the look IDs entered exist and can be queried - and returns the LookML model for each (needed for JSON Schema creation) - """ - looks = [] - for look_id in run_look_ids: - resp = self._request(f"{self.BASE_URL}/looks/{look_id}?fields=model(id),title") - if resp == []: - return ( - [], - f"Unable to find look {look_id}. Verify that you have entered a valid look ID and that you have permission to run it.", - ) - - looks.append((resp[0]["model"]["id"], look_id, resp[0]["title"])) - - return looks, None - - def health_check(self) -> Tuple[bool, str]: - if self._connect_error: - return False, self._connect_error - elif self._run_looks_connect_error: - return False, self._run_looks_connect_error - return True, "" - - @backoff.on_exception(backoff.expo, requests.exceptions.ConnectionError, max_tries=7) - def _request(self, url: str, method: str = "GET", data: dict = None) -> List[dict]: - response = requests.request(method, url, headers=self._headers, json=data) - - if response.status_code == 200: - response_data = response.json() - if isinstance(response_data, list): - return response_data - else: - return [response_data] - return [] - - def _get_run_look_json_schema(self): - """ - Generates a JSON Schema for the run_look endpoint based on the Look IDs - entered in configuration - """ - json_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": True, - "type": "object", - "properties": { - self._get_run_look_key(look_id, look_name): { - "title": look_name, - "properties": {field: self._get_look_field_schema(model, field) for field in self._get_look_fields(look_id)}, - "type": ["null", "object"], - "additionalProperties": False, - } - for (model, look_id, look_name) in self._run_looks - }, - } - return json_schema - - def _get_run_look_key(self, look_id, look_name): - return f"{look_id} - {look_name}" - - def _get_look_field_schema(self, model, field): - """ - For a given LookML model and field, looks up its type and generates - its properties for the run_look endpoint JSON Schema - """ - explore = field.split(".")[0] - - fields = self._get_explore_fields(model, explore) - - field_type = "string" # default to string - for dimension in fields["dimensions"]: - if field == dimension["name"] and dimension["type"] in self._field_type_mapping: - field_type = self._field_type_mapping[dimension["type"]] - for measure in fields["measures"]: - if field == measure["name"]: - # Default to number except for list, date, and yesno - field_type = "number" - if measure["type"] in self._field_type_mapping: - field_type = self._field_type_mapping[measure["type"]] - - if field_type == "datetime": - # no datetime type for JSON Schema - return {"type": ["null", "string"], "format": "date-time"} - - return {"type": ["null", field_type]} - - def _get_explore_fields(self, model, explore): - """ - For a given LookML model and explore, looks up its dimensions/measures - and their types for run_look endpoint JSON Schema generation - """ - if (model, explore) not in self._run_look_explore_fields: - self._run_look_explore_fields[(model, explore)] = self._request( - f"{self.BASE_URL}/lookml_models/{model}/explores/{explore}?fields=fields(dimensions(name,type),measures(name,type))" - )[0]["fields"] - - return self._run_look_explore_fields[(model, explore)] - - def _get_look_fields(self, look_id) -> List[str]: - return self._request(f"{self.BASE_URL}/looks/{look_id}?fields=query(fields)")[0]["query"]["fields"] - - def _get_dashboard_ids(self) -> List[int]: - if not self._dashboard_ids: - self._dashboard_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/dashboards") if isinstance(obj["id"], int)] - return self._dashboard_ids - - def _get_project_ids(self) -> List[int]: - if not self._project_ids: - self._project_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/projects")] - return self._project_ids - - def _get_user_ids(self) -> List[int]: - if not self._user_ids: - self._user_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/users")] - return self._user_ids - - def stream__color_collections(self, fields): - yield from self._request(f"{self.BASE_URL}/color_collections") - - def stream__connections(self, fields): - yield from self._request(f"{self.BASE_URL}/connections") - - def stream__dashboards(self, fields): - dashboards_list = [obj for obj in self._request(f"{self.BASE_URL}/dashboards") if isinstance(obj["id"], int)] - self._dashboard_ids = [obj["id"] for obj in dashboards_list] - self._context_metadata_mapping["dashboards"] = [ - obj["content_metadata_id"] for obj in dashboards_list if isinstance(obj["content_metadata_id"], int) - ] - yield from dashboards_list - - def stream__dashboard_elements(self, fields): - for dashboard_id in self._get_dashboard_ids(): - yield from self._request(f"{self.BASE_URL}/dashboards/{dashboard_id}/dashboard_elements") - - def stream__dashboard_filters(self, fields): - for dashboard_id in self._get_dashboard_ids(): - yield from self._request(f"{self.BASE_URL}/dashboards/{dashboard_id}/dashboard_filters") - - def stream__dashboard_layouts(self, fields): - for dashboard_id in self._get_dashboard_ids(): - yield from self._request(f"{self.BASE_URL}/dashboards/{dashboard_id}/dashboard_layouts") - - def stream__datagroups(self, fields): - yield from self._request(f"{self.BASE_URL}/datagroups") - - def stream__folders(self, fields): - folders_list = self._request(f"{self.BASE_URL}/folders") - self._context_metadata_mapping["folders"] = [ - obj["content_metadata_id"] for obj in folders_list if isinstance(obj["content_metadata_id"], int) - ] - yield from folders_list - - def stream__groups(self, fields): - yield from self._request(f"{self.BASE_URL}/groups") - - def stream__homepages(self, fields): - homepages_list = self._request(f"{self.BASE_URL}/homepages") - self._context_metadata_mapping["homepages"] = [ - obj["content_metadata_id"] for obj in homepages_list if isinstance(obj["content_metadata_id"], int) - ] - yield from homepages_list - - def stream__integration_hubs(self, fields): - yield from self._request(f"{self.BASE_URL}/integration_hubs") - - def stream__integrations(self, fields): - yield from self._request(f"{self.BASE_URL}/integrations") - - def stream__lookml_dashboards(self, fields): - lookml_dashboards_list = [obj for obj in self._request(f"{self.BASE_URL}/dashboards") if isinstance(obj["id"], str)] - yield from lookml_dashboards_list - - def stream__lookml_models(self, fields): - yield from self._request(f"{self.BASE_URL}/lookml_models") - - def stream__looks(self, fields): - looks_list = self._request(f"{self.BASE_URL}/looks") - self._context_metadata_mapping["looks"] = [ - obj["content_metadata_id"] for obj in looks_list if isinstance(obj["content_metadata_id"], int) - ] - yield from looks_list - - def stream__model_sets(self, fields): - yield from self._request(f"{self.BASE_URL}/model_sets") - - def stream__permission_sets(self, fields): - yield from self._request(f"{self.BASE_URL}/permission_sets") - - def stream__permissions(self, fields): - yield from self._request(f"{self.BASE_URL}/permissions") - - def stream__projects(self, fields): - projects_list = self._request(f"{self.BASE_URL}/projects") - self._project_ids = [obj["id"] for obj in projects_list] - yield from projects_list - - def stream__project_files(self, fields): - for project_id in self._get_project_ids(): - yield from self._request(f"{self.BASE_URL}/projects/{project_id}/files") - - def stream__git_branches(self, fields): - for project_id in self._get_project_ids(): - yield from self._request(f"{self.BASE_URL}/projects/{project_id}/git_branches") - - def stream__roles(self, fields): - roles_list = self._request(f"{self.BASE_URL}/roles") - self._role_ids = [obj["id"] for obj in roles_list] - yield from roles_list - - def stream__role_groups(self, fields): - if not self._role_ids: - self._role_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/roles")] - for role_id in self._role_ids: - yield from self._request(f"{self.BASE_URL}/roles/{role_id}/groups") - - def stream__run_looks(self, fields): - for (model, look_id, look_name) in self._run_looks: - yield from [ - {self._get_run_look_key(look_id, look_name): row} for row in self._request(f"{self.BASE_URL}/looks/{look_id}/run/json") - ] - - def stream__scheduled_plans(self, fields): - yield from self._request(f"{self.BASE_URL}/scheduled_plans?all_users=true") - - def stream__spaces(self, fields): - spaces_list = self._request(f"{self.BASE_URL}/spaces") - self._context_metadata_mapping["spaces"] = [ - obj["content_metadata_id"] for obj in spaces_list if isinstance(obj["content_metadata_id"], int) - ] - yield from spaces_list - - def stream__user_attributes(self, fields): - user_attributes_list = self._request(f"{self.BASE_URL}/user_attributes") - self._user_attribute_ids = [obj["id"] for obj in user_attributes_list] - yield from user_attributes_list - - def stream__user_attribute_group_values(self, fields): - if not self._user_attribute_ids: - self._user_attribute_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/user_attributes")] - for user_attribute_id in self._user_attribute_ids: - yield from self._request(f"{self.BASE_URL}/user_attributes/{user_attribute_id}/group_values") - - def stream__user_login_lockouts(self, fields): - yield from self._request(f"{self.BASE_URL}/user_login_lockouts") - - def stream__users(self, fields): - users_list = self._request(f"{self.BASE_URL}/users") - self._user_ids = [obj["id"] for obj in users_list] - yield from users_list - - def stream__user_attribute_values(self, fields): - for user_ids in self._get_user_ids(): - yield from self._request(f"{self.BASE_URL}/users/{user_ids}/attribute_values?all_values=true&include_unset=true") - - def stream__user_sessions(self, fields): - for user_ids in self._get_user_ids(): - yield from self._request(f"{self.BASE_URL}/users/{user_ids}/sessions") - - def stream__versions(self, fields): - yield from self._request(f"{self.BASE_URL}/versions") - - def stream__workspaces(self, fields): - yield from self._request(f"{self.BASE_URL}/workspaces") - - def stream__query_history(self, fields): - request_data = { - "model": "i__looker", - "view": "history", - "fields": [ - "query.id", - "history.created_date", - "query.model", - "query.view", - "space.id", - "look.id", - "dashboard.id", - "user.id", - "history.query_run_count", - "history.total_runtime", - ], - "filters": {"query.model": "-EMPTY", "history.runtime": "NOT NULL", "user.is_looker": "No"}, - "sorts": [ - "-history.created_date" "query.id", - ], - } - history_list = self._request(f"{self.BASE_URL}/queries/run/json?limit=10000", method="POST", data=request_data) - for history_data in history_list: - yield {k.replace(".", "_"): v for k, v in history_data.items()} - - def stream__content_metadata(self, fields): - yield from self._metadata_processing(f"{self.BASE_URL}/content_metadata/") - - def stream__content_metadata_access(self, fields): - yield from self._metadata_processing(f"{self.BASE_URL}/content_metadata_access?content_metadata_id=") - - def _metadata_processing(self, url: str): - content_metadata_id_list = [] - for metadata_main_obj, ids in self._context_metadata_mapping.items(): - if not ids: - metadata_id_list = [ - obj["content_metadata_id"] - for obj in self._request(f"{self.BASE_URL}/{metadata_main_obj}") - if isinstance(obj["content_metadata_id"], int) - ] - self._context_metadata_mapping[metadata_main_obj] = metadata_id_list - content_metadata_id_list += metadata_id_list - else: - content_metadata_id_list += ids - - for metadata_id in set(content_metadata_id_list): - yield from self._request(f"{url}{metadata_id}") diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/color_collections.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/color_collections.json deleted file mode 100644 index b2acf2691a8ce..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/color_collections.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "categoricalPalettes": { - "items": { - "properties": { - "colors": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "id": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "divergingPalettes": { - "items": { - "properties": { - "id": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "stops": { - "items": { - "properties": { - "color": { - "type": ["null", "string"] - }, - "offset": { - "multipleOf": 1e-16, - "type": ["null", "number"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "type": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "id": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "sequentialPalettes": { - "items": { - "properties": { - "id": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "stops": { - "items": { - "properties": { - "color": { - "type": ["null", "string"] - }, - "offset": { - "multipleOf": 1e-16, - "type": ["null", "number"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "type": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/connections.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/connections.json deleted file mode 100644 index af56601f5a0c5..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/connections.json +++ /dev/null @@ -1,199 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "after_connect_statements": { - "type": ["null", "string"] - }, - "certificate": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"] - }, - "database": { - "type": ["null", "string"] - }, - "db_timezone": { - "type": ["null", "string"] - }, - "dialect": { - "properties": { - "automatically_run_sql_runner_snippets": { - "type": ["null", "boolean"] - }, - "connection_tests": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "label": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "persistent_table_distkey": { - "type": ["null", "string"] - }, - "persistent_table_indexes": { - "type": ["null", "string"] - }, - "persistent_table_sortkeys": { - "type": ["null", "string"] - }, - "supports_cost_estimate": { - "type": ["null", "boolean"] - }, - "supports_inducer": { - "type": ["null", "boolean"] - }, - "supports_streaming": { - "type": ["null", "boolean"] - }, - "supports_upload_tables": { - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"] - }, - "dialect_name": { - "type": ["null", "string"] - }, - "example": { - "type": ["null", "boolean"] - }, - "file_type": { - "type": ["null", "string"] - }, - "host": { - "type": ["null", "string"] - }, - "jdbc_additional_params": { - "type": ["null", "string"] - }, - "last_reap_at": { - "type": ["null", "integer"] - }, - "last_regen_at": { - "type": ["null", "integer"] - }, - "maintenance_cron": { - "type": ["null", "string"] - }, - "max_billing_gigabytes": { - "multipleOf": 1e-8, - "type": ["null", "number"] - }, - "max_connections": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "password": { - "type": ["null", "string"] - }, - "pdt_context_override": { - "properties": { - "after_connect_statements": { - "type": ["null", "string"] - }, - "certificate": { - "type": ["null", "string"] - }, - "context": { - "type": ["null", "string"] - }, - "database": { - "type": ["null", "string"] - }, - "file_type": { - "type": ["null", "string"] - }, - "has_password": { - "type": ["null", "boolean"] - }, - "host": { - "type": ["null", "string"] - }, - "jdbc_additional_params": { - "type": ["null", "string"] - }, - "password": { - "type": ["null", "string"] - }, - "port": { - "type": ["null", "string"] - }, - "schema": { - "type": ["null", "string"] - }, - "username": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "pool_timeout": { - "type": ["null", "integer"] - }, - "port": { - "type": ["null", "string"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "schema": { - "type": ["null", "string"] - }, - "snippets": { - "items": { - "properties": { - "label": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "sql": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "sql_runner_precache_tables": { - "type": ["null", "boolean"] - }, - "ssl": { - "type": ["null", "boolean"] - }, - "tmp_db_name": { - "type": ["null", "string"] - }, - "user_attribute_fields": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "user_db_credentials": { - "type": ["null", "boolean"] - }, - "user_id": { - "type": ["null", "integer"] - }, - "username": { - "type": ["null", "string"] - }, - "uses_oauth": { - "type": ["null", "boolean"] - }, - "verify_ssl": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/content_metadata.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/content_metadata.json deleted file mode 100644 index 87939f0234459..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/content_metadata.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "content_type": { - "type": ["null", "string"] - }, - "dashboard_id": { - "type": ["null", "integer"] - }, - "folder_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "inheriting_id": { - "type": ["null", "integer"] - }, - "inherits": { - "type": ["null", "boolean"] - }, - "look_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - }, - "slug": { - "type": ["null", "string"] - }, - "space_id": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/content_metadata_access.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/content_metadata_access.json deleted file mode 100644 index 4079402a9e129..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/content_metadata_access.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "content_metadata_id": { - "type": ["null", "integer"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "permission_type": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_elements.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_elements.json deleted file mode 100644 index 309ad9355ed12..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_elements.json +++ /dev/null @@ -1,638 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "alert_count": { - "type": ["null", "integer"] - }, - "body_text": { - "type": ["null", "string"] - }, - "body_text_as_html": { - "type": ["null", "string"] - }, - "dashboard_id": { - "type": ["null", "integer"] - }, - "edit_uri": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "look": { - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleted": { - "type": ["null", "boolean"] - }, - "deleted_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleter_id": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "embed_url": { - "type": ["null", "string"] - }, - "excel_file_url": { - "type": ["null", "string"] - }, - "favorite_count": { - "type": ["null", "integer"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "folder_id": { - "type": ["null", "string"] - }, - "google_spreadsheet_formula": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "image_embed_url": { - "type": ["null", "string"] - }, - "is_run_on_load": { - "type": ["null", "boolean"] - }, - "last_accessed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_updater_id": { - "type": ["null", "string"] - }, - "last_viewed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "public": { - "type": ["null", "boolean"] - }, - "public_slug": { - "type": ["null", "string"] - }, - "public_url": { - "type": ["null", "string"] - }, - "query": { - "properties": { - "client_id": { - "type": ["null", "string"] - }, - "column_limit": { - "type": ["null", "string"] - }, - "dynamic_fields": { - "type": ["null", "string"] - }, - "expanded_share_url": { - "type": ["null", "string"] - }, - "fields": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "fill_fields": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "filter_config": { - "properties": {}, - "type": ["null", "object"] - }, - "filter_expression": { - "type": ["null", "string"] - }, - "filters": { - "properties": {}, - "type": ["null", "object"] - }, - "has_table_calculations": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "string"] - }, - "limit": { - "type": ["null", "string"] - }, - "model": { - "type": ["null", "string"] - }, - "pivots": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "row_total": { - "type": ["null", "string"] - }, - "runtime": { - "type": ["null", "number"] - }, - "share_url": { - "type": ["null", "string"] - }, - "slug": { - "type": ["null", "string"] - }, - "sorts": { - "items": {}, - "type": ["null", "array"] - }, - "subtotals": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "total": { - "type": ["null", "boolean"] - }, - "url": { - "type": ["null", "string"] - }, - "view": { - "type": ["null", "string"] - }, - "vis_config": { - "properties": {}, - "type": ["null", "object"] - }, - "visible_ui_sections": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_id": { - "type": ["null", "string"] - }, - "short_url": { - "type": ["null", "string"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "space_id": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "updated_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "user": { - "properties": { - "id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "user_id": { - "type": ["null", "string"] - }, - "view_count": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "look_id": { - "type": ["null", "string"] - }, - "lookml_link_id": { - "type": ["null", "string"] - }, - "merge_result_id": { - "type": ["null", "string"] - }, - "note_display": { - "type": ["null", "string"] - }, - "note_state": { - "type": ["null", "string"] - }, - "note_text": { - "type": ["null", "string"] - }, - "note_text_as_html": { - "type": ["null", "string"] - }, - "query": { - "properties": { - "client_id": { - "type": ["null", "string"] - }, - "column_limit": { - "type": ["null", "string"] - }, - "dynamic_fields": { - "type": ["null", "string"] - }, - "expanded_share_url": { - "type": ["null", "string"] - }, - "fields": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "fill_fields": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "filter_config": { - "properties": {}, - "type": ["null", "object"] - }, - "filter_expression": { - "type": ["null", "string"] - }, - "filters": { - "properties": {}, - "type": ["null", "object"] - }, - "has_table_calculations": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "string"] - }, - "limit": { - "type": ["null", "string"] - }, - "model": { - "type": ["null", "string"] - }, - "pivots": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "row_total": { - "type": ["null", "string"] - }, - "runtime": { - "type": ["null", "number"] - }, - "share_url": { - "type": ["null", "string"] - }, - "slug": { - "type": ["null", "string"] - }, - "sorts": { - "items": {}, - "type": ["null", "array"] - }, - "subtotals": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "total": { - "type": ["null", "boolean"] - }, - "url": { - "type": ["null", "string"] - }, - "view": { - "type": ["null", "string"] - }, - "vis_config": { - "properties": {}, - "type": ["null", "object"] - }, - "visible_ui_sections": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_id": { - "type": ["null", "string"] - }, - "refresh_interval": { - "type": ["null", "string"] - }, - "refresh_interval_to_i": { - "type": ["null", "integer"] - }, - "result_maker": { - "properties": { - "dynamic_fields": { - "type": ["null", "string"] - }, - "filterables": { - "items": { - "properties": { - "listen": { - "items": { - "properties": { - "dashboard_filter_name": { - "type": ["null", "string"] - }, - "field": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "model": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "view": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "id": { - "type": ["null", "integer"] - }, - "merge_result_id": { - "type": ["null", "integer"] - }, - "query": { - "properties": { - "client_id": { - "type": ["null", "string"] - }, - "column_limit": { - "type": ["null", "string"] - }, - "dynamic_fields": { - "type": ["null", "string"] - }, - "expanded_share_url": { - "type": ["null", "string"] - }, - "fields": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "fill_fields": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "filter_config": { - "properties": {}, - "type": ["null", "object"] - }, - "filter_expression": { - "type": ["null", "string"] - }, - "filters": { - "properties": {}, - "type": ["null", "object"] - }, - "has_table_calculations": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "limit": { - "type": ["null", "string"] - }, - "model": { - "type": ["null", "string"] - }, - "pivots": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "row_total": { - "type": ["null", "string"] - }, - "runtime": { - "type": ["null", "number"] - }, - "share_url": { - "type": ["null", "string"] - }, - "slug": { - "type": ["null", "string"] - }, - "sorts": { - "items": {}, - "type": ["null", "array"] - }, - "subtotals": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "total": { - "type": ["null", "boolean"] - }, - "url": { - "type": ["null", "string"] - }, - "view": { - "type": ["null", "string"] - }, - "vis_config": { - "properties": {}, - "type": ["null", "object"] - }, - "visible_ui_sections": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_id": { - "type": ["null", "integer"] - }, - "sorts": { - "items": {}, - "type": ["null", "array"] - }, - "total": { - "type": ["null", "boolean"] - }, - "vis_config": { - "properties": {}, - "type": ["null", "object"] - } - }, - "type": ["null", "object"] - }, - "result_maker_id": { - "type": ["null", "integer"] - }, - "subtitle_text": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "title_hidden": { - "type": ["null", "boolean"] - }, - "title_text": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_filters.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_filters.json deleted file mode 100644 index 4af72fec76975..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_filters.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "allow_multiple_values": { - "type": ["null", "boolean"] - }, - "dashboard_id": { - "type": ["null", "integer"] - }, - "default_value": { - "type": ["null", "string"] - }, - "dimension": { - "type": ["null", "string"] - }, - "explore": { - "type": ["null", "string"] - }, - "field": { - "properties": {}, - "type": ["null", "object"] - }, - "id": { - "type": ["null", "integer"] - }, - "listens_to_filters": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "model": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "required": { - "type": ["null", "boolean"] - }, - "row": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "ui_config": { - "properties": {}, - "type": ["null", "object"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_layouts.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_layouts.json deleted file mode 100644 index b92b40944c236..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboard_layouts.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "active": { - "type": ["null", "boolean"] - }, - "column_width": { - "type": ["null", "integer"] - }, - "dashboard_id": { - "type": ["null", "integer"] - }, - "dashboard_layout_components": { - "items": { - "properties": { - "column": { - "type": ["null", "integer"] - }, - "dashboard_element_id": { - "type": ["null", "integer"] - }, - "dashboard_layout_id": { - "type": ["null", "integer"] - }, - "deleted": { - "type": ["null", "boolean"] - }, - "element_title": { - "type": ["null", "string"] - }, - "element_title_hidden": { - "type": ["null", "boolean"] - }, - "height": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "row": { - "type": ["null", "integer"] - }, - "vis_type": { - "type": ["null", "string"] - }, - "width": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "dashboard_title": { - "type": ["null", "string"] - }, - "deleted": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "string"] - }, - "width": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboards.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboards.json deleted file mode 100644 index 1139ce11ecb17..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/dashboards.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "readonly": { - "type": ["null", "boolean"] - }, - "refresh_interval": { - "type": ["null", "string"] - }, - "refresh_interval_to_i": { - "type": ["null", "integer"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "title": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/datagroups.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/datagroups.json deleted file mode 100644 index ed7dca3251007..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/datagroups.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "created_at": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "model_name": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "stale_before": { - "type": ["null", "integer"] - }, - "trigger_check_at": { - "type": ["null", "integer"] - }, - "trigger_error": { - "type": ["null", "string"] - }, - "trigger_value": { - "type": ["null", "string"] - }, - "triggered_at": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/folders.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/folders.json deleted file mode 100644 index 46a8003379b3d..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/folders.json +++ /dev/null @@ -1,550 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "dashboards": { - "items": { - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "readonly": { - "type": ["null", "boolean"] - }, - "refresh_interval": { - "type": ["null", "string"] - }, - "refresh_interval_to_i": { - "type": ["null", "integer"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "title": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "looks": { - "items": { - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "dashboards": { - "items": { - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "readonly": { - "type": ["null", "boolean"] - }, - "refresh_interval": { - "type": ["null", "string"] - }, - "refresh_interval_to_i": { - "type": ["null", "integer"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "title": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "deleted": { - "type": ["null", "boolean"] - }, - "deleted_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleter_id": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "embed_url": { - "type": ["null", "string"] - }, - "excel_file_url": { - "type": ["null", "string"] - }, - "favorite_count": { - "type": ["null", "integer"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "folder_id": { - "type": ["null", "string"] - }, - "google_spreadsheet_formula": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "image_embed_url": { - "type": ["null", "string"] - }, - "is_run_on_load": { - "type": ["null", "boolean"] - }, - "last_accessed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_updater_id": { - "type": ["null", "string"] - }, - "last_viewed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "public": { - "type": ["null", "boolean"] - }, - "public_slug": { - "type": ["null", "string"] - }, - "public_url": { - "type": ["null", "string"] - }, - "query_id": { - "type": ["null", "string"] - }, - "short_url": { - "type": ["null", "string"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "space_id": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "updated_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "user": { - "properties": { - "id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "user_id": { - "type": ["null", "string"] - }, - "view_count": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/git_branches.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/git_branches.json deleted file mode 100644 index 8ad5151733b35..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/git_branches.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ahead_count": { - "type": ["null", "integer"] - }, - "behind_count": { - "type": ["null", "integer"] - }, - "commit_at": { - "type": ["null", "integer"] - }, - "error": { - "type": ["null", "string"] - }, - "is_local": { - "type": ["null", "boolean"] - }, - "is_production": { - "type": ["null", "boolean"] - }, - "is_remote": { - "type": ["null", "boolean"] - }, - "message": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "owner_name": { - "type": ["null", "string"] - }, - "personal": { - "type": ["null", "boolean"] - }, - "project_id": { - "type": ["null", "string"] - }, - "readonly": { - "type": ["null", "boolean"] - }, - "ref": { - "type": ["null", "string"] - }, - "remote": { - "type": ["null", "string"] - }, - "remote_name": { - "type": ["null", "string"] - }, - "remote_ref": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/groups.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/groups.json deleted file mode 100644 index 800514358ebdc..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/groups.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "can_add_to_content_metadata": { - "type": ["null", "boolean"] - }, - "contains_current_user": { - "type": ["null", "boolean"] - }, - "external_group_id": { - "type": ["null", "string"] - }, - "externally_managed": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "include_by_default": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "user_count": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/homepages.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/homepages.json deleted file mode 100644 index 800741622a9f1..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/homepages.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "content_metadata_id": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleted_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "homepage_sections": { - "items": { - "properties": { - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleted_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "detail_url": { - "type": ["null", "string"] - }, - "homepage_id": { - "type": ["null", "string"] - }, - "homepage_items": { - "items": { - "properties": { - "content_created_by": { - "type": ["null", "string"] - }, - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "content_updated_at": { - "type": ["null", "string"] - }, - "custom_description": { - "type": ["null", "string"] - }, - "custom_image_data_base64": { - "type": ["null", "string"] - }, - "custom_image_url": { - "type": ["null", "string"] - }, - "custom_title": { - "type": ["null", "string"] - }, - "custom_url": { - "type": ["null", "string"] - }, - "dashboard_id": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "favorite_count": { - "type": ["null", "integer"] - }, - "homepage_section_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "image_url": { - "type": ["null", "string"] - }, - "location": { - "type": ["null", "string"] - }, - "look_id": { - "type": ["null", "string"] - }, - "lookml_dashboard_id": { - "type": ["null", "string"] - }, - "order": { - "type": ["null", "integer"] - }, - "section_fetch_time": { - "format": "float", - "type": ["null", "number"] - }, - "title": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "use_custom_description": { - "type": ["null", "boolean"] - }, - "use_custom_image": { - "type": ["null", "boolean"] - }, - "use_custom_title": { - "type": ["null", "boolean"] - }, - "use_custom_url": { - "type": ["null", "boolean"] - }, - "view_count": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "id": { - "type": ["null", "string"] - }, - "is_header": { - "type": ["null", "boolean"] - }, - "item_order": { - "items": { - "type": ["null", "integer"] - }, - "type": ["null", "array"] - }, - "title": { - "type": ["null", "string"] - }, - "updated_at": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "id": { - "type": ["null", "string"] - }, - "primary_homepage": { - "type": ["null", "boolean"] - }, - "section_order": { - "items": { - "type": ["null", "integer"] - }, - "type": ["null", "array"] - }, - "title": { - "type": ["null", "string"] - }, - "updated_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/integration_hubs.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/integration_hubs.json deleted file mode 100644 index 77cd556e6c400..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/integration_hubs.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "authorization_token": { - "type": ["null", "string"] - }, - "fetch_error_message": { - "type": ["null", "string"] - }, - "has_authorization_token": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - }, - "legal_agreement_required": { - "type": ["null", "boolean"] - }, - "legal_agreement_signed": { - "type": ["null", "boolean"] - }, - "legal_agreement_text": { - "type": ["null", "string"] - }, - "official": { - "type": ["null", "boolean"] - }, - "url": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/integrations.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/integrations.json deleted file mode 100644 index 6febdc8f64f8c..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/integrations.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "delegate_oauth": { - "type": ["null", "boolean"] - }, - "description": { - "type": ["null", "string"] - }, - "enabled": { - "type": ["null", "boolean"] - }, - "icon_url": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "installed_delegate_oauth": { - "type": ["null", "string"] - }, - "integration_hub_id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - }, - "params": { - "items": { - "properties": { - "delegate_oauth_url": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "has_value": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "per_user": { - "type": ["null", "boolean"] - }, - "required": { - "type": ["null", "boolean"] - }, - "sensitive": { - "type": ["null", "boolean"] - }, - "user_attribute_name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "required_fields": { - "items": { - "properties": { - "all_tags": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "any_tag": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "tag": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "supported_action_types": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "supported_download_settings": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "supported_formats": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "supported_formattings": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "supported_visualization_formattings": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "uses_oauth": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/lookml_dashboards.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/lookml_dashboards.json deleted file mode 100644 index 0fcf5526c185e..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/lookml_dashboards.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "string"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "readonly": { - "type": ["null", "boolean"] - }, - "refresh_interval": { - "type": ["null", "string"] - }, - "refresh_interval_to_i": { - "type": ["null", "integer"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "title": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/lookml_models.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/lookml_models.json deleted file mode 100644 index d1296e90496c0..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/lookml_models.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "allowed_db_connection_names": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "explores": { - "items": { - "properties": { - "description": { - "type": ["null", "string"] - }, - "group_label": { - "type": ["null", "string"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "has_content": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "project_name": { - "type": ["null", "string"] - }, - "unlimited_db_connections": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/looks.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/looks.json deleted file mode 100644 index 7597502e67b7c..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/looks.json +++ /dev/null @@ -1,208 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleted": { - "type": ["null", "boolean"] - }, - "deleted_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleter_id": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "embed_url": { - "type": ["null", "string"] - }, - "excel_file_url": { - "type": ["null", "string"] - }, - "favorite_count": { - "type": ["null", "integer"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "folder_id": { - "type": ["null", "string"] - }, - "google_spreadsheet_formula": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "image_embed_url": { - "type": ["null", "string"] - }, - "is_run_on_load": { - "type": ["null", "boolean"] - }, - "last_accessed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_updater_id": { - "type": ["null", "string"] - }, - "last_viewed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "public": { - "type": ["null", "boolean"] - }, - "public_slug": { - "type": ["null", "string"] - }, - "public_url": { - "type": ["null", "string"] - }, - "query_id": { - "type": ["null", "string"] - }, - "short_url": { - "type": ["null", "string"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "space_id": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "updated_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "user": { - "properties": { - "id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "user_id": { - "type": ["null", "string"] - }, - "view_count": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/model_sets.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/model_sets.json deleted file mode 100644 index dc1601a79fe33..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/model_sets.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "all_access": { - "type": ["null", "boolean"] - }, - "built_in": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "models": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/permission_sets.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/permission_sets.json deleted file mode 100644 index 3ea73e967f973..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/permission_sets.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "all_access": { - "type": ["null", "boolean"] - }, - "built_in": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "permissions": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "url": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/permissions.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/permissions.json deleted file mode 100644 index 12d7f54da555c..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/permissions.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "parent": { - "type": ["null", "string"] - }, - "permission": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/project_files.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/project_files.json deleted file mode 100644 index f32fc4e55e172..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/project_files.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "editable": { - "type": ["null", "boolean"] - }, - "extension": { - "type": ["null", "string"] - }, - "git_status": { - "properties": { - "action": { - "type": ["null", "string"] - }, - "conflict": { - "type": ["null", "boolean"] - }, - "revertable": { - "type": ["null", "boolean"] - }, - "text": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "id": { - "type": ["null", "string"] - }, - "mime_type": { - "type": ["null", "string"] - }, - "path": { - "type": ["null", "string"] - }, - "project_id": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/projects.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/projects.json deleted file mode 100644 index 4e1a2a8f5fa46..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/projects.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "allow_warnings": { - "type": ["null", "boolean"] - }, - "deploy_secret": { - "type": ["null", "string"] - }, - "folders_enabled": { - "type": ["null", "boolean"] - }, - "git_password": { - "type": ["null", "string"] - }, - "git_password_user_attribute": { - "type": ["null", "string"] - }, - "git_remote_url": { - "type": ["null", "string"] - }, - "git_service_name": { - "type": ["null", "string"] - }, - "git_username": { - "type": ["null", "string"] - }, - "git_username_user_attribute": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_example": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "pull_request_mode": { - "type": ["null", "string"] - }, - "unset_deploy_secret": { - "type": ["null", "boolean"] - }, - "uses_git": { - "type": ["null", "boolean"] - }, - "validation_required": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/query_history.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/query_history.json index f539dea5207ee..ebe0a1c91511f 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/query_history.json +++ b/airbyte-integrations/connectors/source-looker/source_looker/schemas/query_history.json @@ -3,35 +3,70 @@ "type": "object", "properties": { "query_id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "history_created_date": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ], + "format": "date" + }, + "history_created_time": { + "type": "string", + "format": "date-time" }, "query_model": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "query_view": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "space_id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "look_id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "dashboard_id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "user_id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "history_query_run_count": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "history_total_runtime": { "multipleOf": 1e-20, - "type": ["null", "number"] + "type": [ + "null", + "number" + ] } } } diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/role_groups.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/role_groups.json deleted file mode 100644 index 3910a148a9ea2..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/role_groups.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "can_add_to_content_metadata": { - "type": ["null", "boolean"] - }, - "contains_current_user": { - "type": ["null", "boolean"] - }, - "external_group_id": { - "type": ["null", "string"] - }, - "externally_managed": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "include_by_default": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "role_id": { - "type": ["null", "string"] - }, - "user_count": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/roles.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/roles.json deleted file mode 100644 index c3cb5b0814a9e..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/roles.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "model_set": { - "properties": { - "all_access": { - "type": ["null", "boolean"] - }, - "built_in": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "models": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "model_set_id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "permission_set": { - "properties": { - "all_access": { - "type": ["null", "boolean"] - }, - "built_in": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "permissions": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "permission_set_id": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "users_url": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/run_looks.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/run_looks.json deleted file mode 100644 index b980c7b08c29d..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/run_looks.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "comment": "This schema gets created in client.py, but we need a placeholder for the super() method to work" -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/scheduled_plans.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/scheduled_plans.json deleted file mode 100644 index 2f01ff0e672b1..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/scheduled_plans.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "color_theme": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "crontab": { - "type": ["null", "string"] - }, - "dashboard_filters": { - "type": ["null", "string"] - }, - "dashboard_id": { - "type": ["null", "string"] - }, - "datagroup": { - "type": ["null", "string"] - }, - "embed": { - "type": ["null", "boolean"] - }, - "enabled": { - "type": ["null", "boolean"] - }, - "filters_string": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "include_links": { - "type": ["null", "boolean"] - }, - "last_run_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "long_tables": { - "type": ["null", "boolean"] - }, - "look_id": { - "type": ["null", "string"] - }, - "lookml_dashboard_id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "next_run_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "pdf_landscape": { - "type": ["null", "boolean"] - }, - "pdf_paper_size": { - "type": ["null", "string"] - }, - "query_id": { - "type": ["null", "string"] - }, - "require_change": { - "type": ["null", "boolean"] - }, - "require_no_results": { - "type": ["null", "boolean"] - }, - "require_results": { - "type": ["null", "boolean"] - }, - "run_as_recipient": { - "type": ["null", "boolean"] - }, - "run_once": { - "type": ["null", "boolean"] - }, - "scheduled_plan_destination": { - "items": { - "properties": { - "address": { - "type": ["null", "string"] - }, - "apply_formatting": { - "type": ["null", "boolean"] - }, - "apply_vis": { - "type": ["null", "boolean"] - }, - "format": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "looker_recipient": { - "type": ["null", "boolean"] - }, - "message": { - "type": ["null", "string"] - }, - "parameters": { - "type": ["null", "string"] - }, - "scheduled_plan_id": { - "type": ["null", "string"] - }, - "secret_parameters": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "send_all_results": { - "type": ["null", "boolean"] - }, - "timezone": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "updated_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "user": { - "properties": { - "avatar_url": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "user_id": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/spaces.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/spaces.json deleted file mode 100644 index 46a8003379b3d..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/spaces.json +++ /dev/null @@ -1,550 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "dashboards": { - "items": { - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "readonly": { - "type": ["null", "boolean"] - }, - "refresh_interval": { - "type": ["null", "string"] - }, - "refresh_interval_to_i": { - "type": ["null", "integer"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "title": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "looks": { - "items": { - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "dashboards": { - "items": { - "properties": { - "content_favorite_id": { - "type": ["null", "string"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "query_timezone": { - "type": ["null", "string"] - }, - "readonly": { - "type": ["null", "boolean"] - }, - "refresh_interval": { - "type": ["null", "string"] - }, - "refresh_interval_to_i": { - "type": ["null", "integer"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "title": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "deleted": { - "type": ["null", "boolean"] - }, - "deleted_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "deleter_id": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "embed_url": { - "type": ["null", "string"] - }, - "excel_file_url": { - "type": ["null", "string"] - }, - "favorite_count": { - "type": ["null", "integer"] - }, - "folder": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "folder_id": { - "type": ["null", "string"] - }, - "google_spreadsheet_formula": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "image_embed_url": { - "type": ["null", "string"] - }, - "is_run_on_load": { - "type": ["null", "boolean"] - }, - "last_accessed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_updater_id": { - "type": ["null", "string"] - }, - "last_viewed_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "model": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "public": { - "type": ["null", "boolean"] - }, - "public_slug": { - "type": ["null", "string"] - }, - "public_url": { - "type": ["null", "string"] - }, - "query_id": { - "type": ["null", "string"] - }, - "short_url": { - "type": ["null", "string"] - }, - "space": { - "properties": { - "child_count": { - "type": ["null", "integer"] - }, - "content_metadata_id": { - "type": ["null", "integer"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "external_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_embed": { - "type": ["null", "boolean"] - }, - "is_embed_shared_root": { - "type": ["null", "boolean"] - }, - "is_embed_users_root": { - "type": ["null", "boolean"] - }, - "is_personal": { - "type": ["null", "boolean"] - }, - "is_personal_descendant": { - "type": ["null", "boolean"] - }, - "is_shared_root": { - "type": ["null", "boolean"] - }, - "is_users_root": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "space_id": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "updated_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "user": { - "properties": { - "id": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "user_id": { - "type": ["null", "string"] - }, - "view_count": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "name": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attribute_group_values.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attribute_group_values.json deleted file mode 100644 index 4788e5e7b0c6e..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attribute_group_values.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "group_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "rank": { - "type": ["null", "integer"] - }, - "user_attribute_id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "value_is_hidden": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attributes.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attributes.json deleted file mode 100644 index 1902c27b3217f..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_attributes.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "default_value": { - "type": ["null", "string"] - }, - "hidden_value_domain_whitelist": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_permanent": { - "type": ["null", "boolean"] - }, - "is_system": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "user_can_edit": { - "type": ["null", "boolean"] - }, - "user_can_view": { - "type": ["null", "boolean"] - }, - "value_is_hidden": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_login_lockouts.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_login_lockouts.json deleted file mode 100644 index fd02699402b2a..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_login_lockouts.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "auth_type": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fail_count": { - "type": ["null", "integer"] - }, - "full_name": { - "type": ["null", "string"] - }, - "ip": { - "type": ["null", "integer"] - }, - "key": { - "type": ["null", "string"] - }, - "lockout_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "remote_id": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_sessions.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_sessions.json deleted file mode 100644 index d1208c76e4c6f..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/user_sessions.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "browser": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"] - }, - "credentials_type": { - "type": ["null", "string"] - }, - "expires_at": { - "type": ["null", "string"] - }, - "extended_at": { - "type": ["null", "string"] - }, - "extended_count": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "ip_address": { - "type": ["null", "string"] - }, - "operating_system": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "sudo_user_id": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/users.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/users.json deleted file mode 100644 index a0346ad874d0b..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/users.json +++ /dev/null @@ -1,384 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "avatar_url": { - "type": ["null", "string"] - }, - "avatar_url_without_sizing": { - "type": ["null", "string"] - }, - "credentials_api3": { - "items": { - "properties": { - "client_id": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "credentials_email": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "forced_password_reset_at_next_login": { - "type": ["null", "boolean"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "logged_in_at": { - "type": ["null", "string"] - }, - "password_reset_url": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "user_url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "credentials_embed": { - "items": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "external_group_id": { - "type": ["null", "string"] - }, - "external_user_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "logged_in_at": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "credentials_google": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "domain": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "google_user_id": { - "type": ["null", "string"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "logged_in_at": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "credentials_ldap": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "ldap_dn": { - "type": ["null", "string"] - }, - "ldap_id": { - "type": ["null", "string"] - }, - "logged_in_at": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "credentials_looker_openid": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "logged_in_at": { - "type": ["null", "string"] - }, - "logged_in_ip": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "user_url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "credentials_oidc": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "logged_in_at": { - "type": ["null", "string"] - }, - "oidc_user_id": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "credentials_saml": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "logged_in_at": { - "type": ["null", "string"] - }, - "saml_user_id": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "credentials_totp": { - "properties": { - "created_at": { - "type": ["null", "string"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "verified": { - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"] - }, - "display_name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "embed_group_space_id": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "group_ids": { - "items": { - "type": ["null", "integer"] - }, - "type": ["null", "array"] - }, - "home_folder_id": { - "type": ["null", "string"] - }, - "home_space_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_disabled": { - "type": ["null", "boolean"] - }, - "last_name": { - "type": ["null", "string"] - }, - "locale": { - "type": ["null", "string"] - }, - "looker_versions": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "models_dir_validated": { - "type": ["null", "boolean"] - }, - "personal_folder_id": { - "type": ["null", "integer"] - }, - "personal_space_id": { - "type": ["null", "integer"] - }, - "presumed_looker_employee": { - "type": ["null", "boolean"] - }, - "role_ids": { - "items": { - "type": ["null", "integer"] - }, - "type": ["null", "array"] - }, - "roles_externally_managed": { - "type": ["null", "boolean"] - }, - "sessions": { - "items": { - "properties": { - "browser": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"] - }, - "credentials_type": { - "type": ["null", "string"] - }, - "expires_at": { - "type": ["null", "string"] - }, - "extended_at": { - "type": ["null", "string"] - }, - "extended_count": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "ip_address": { - "type": ["null", "string"] - }, - "operating_system": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "sudo_user_id": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "ui_state": { - "properties": {}, - "type": ["null", "object"] - }, - "url": { - "type": ["null", "string"] - }, - "verified_looker_employee": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/versions.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/versions.json deleted file mode 100644 index 6f77ab6b2ef94..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/versions.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "current_version": { - "properties": { - "full_version": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "swagger_url": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "looker_release_version": { - "type": ["null", "string"] - }, - "supported_versions": { - "items": { - "properties": { - "full_version": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "swagger_url": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/schemas/workspaces.json b/airbyte-integrations/connectors/source-looker/source_looker/schemas/workspaces.json deleted file mode 100644 index a6360a7bb9099..0000000000000 --- a/airbyte-integrations/connectors/source-looker/source_looker/schemas/workspaces.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "projects": { - "items": { - "properties": { - "allow_warnings": { - "type": ["null", "boolean"] - }, - "deploy_secret": { - "type": ["null", "string"] - }, - "folders_enabled": { - "type": ["null", "boolean"] - }, - "git_password": { - "type": ["null", "string"] - }, - "git_password_user_attribute": { - "type": ["null", "string"] - }, - "git_remote_url": { - "type": ["null", "string"] - }, - "git_service_name": { - "type": ["null", "string"] - }, - "git_username": { - "type": ["null", "string"] - }, - "git_username_user_attribute": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "is_example": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "pull_request_mode": { - "type": ["null", "string"] - }, - "unset_deploy_secret": { - "type": ["null", "boolean"] - }, - "uses_git": { - "type": ["null", "boolean"] - }, - "validation_required": { - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - } - } -} diff --git a/airbyte-integrations/connectors/source-looker/source_looker/source.py b/airbyte-integrations/connectors/source-looker/source_looker/source.py index de917cde8be6d..51facdb7cf9b9 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/source.py +++ b/airbyte-integrations/connectors/source-looker/source_looker/source.py @@ -2,11 +2,117 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +from typing import Any, List, Mapping, Optional, Tuple -from airbyte_cdk.sources.deprecated.base_source import BaseSource +import pendulum +import requests +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from .client import Client +from .streams import API_VERSION, ContentMetadata, Dashboards, LookerException, LookerStream, QueryHistory, RunLooks, SwaggerParser -class SourceLooker(BaseSource): - client_class = Client +class CustomTokenAuthenticator(TokenAuthenticator): + def __init__(self, domain: str, client_id: str, client_secret: str): + self._domain, self._client_id, self._client_secret = domain, client_id, client_secret + super().__init__(None) + + self._access_token = None + self._token_expiry_date = pendulum.now() + + def update_access_token(self) -> Optional[str]: + headers = {"Content-Type": "application/x-www-form-urlencoded"} + url = f"https://{self._domain}/api/{API_VERSION}/login" + try: + resp = requests.post(url=url, headers=headers, data=f"client_id={self._client_id}&client_secret={self._client_secret}") + if resp.status_code != 200: + return "Unable to connect to the Looker API. Please check your credentials." + except ConnectionError as error: + return str(error) + data = resp.json() + self._access_token = data["access_token"] + self._token_expiry_date = pendulum.now().add(seconds=data["expires_in"]) + return None + + def get_auth_header(self) -> Mapping[str, Any]: + if self._token_expiry_date < pendulum.now(): + err = self.update_access_token() + if err: + raise LookerException(f"auth error: {err}") + return {"Authorization": f"token {self._access_token}"} + + +class SourceLooker(AbstractSource): + """ + Source Intercom fetch data from messaging platform. + """ + + def get_authenticator(self, config: Mapping[str, Any]) -> CustomTokenAuthenticator: + return CustomTokenAuthenticator(domain=config["domain"], client_id=config["client_id"], client_secret=config["client_secret"]) + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + authenticator = self.get_authenticator(config) + err = authenticator.update_access_token() + if err: + AirbyteLogger().error("auth error: {err}") + return False, err + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + base_args = { + "authenticator": self.get_authenticator(config), + "domain": config["domain"], + } + args = dict(swagger_parser=SwaggerParser(domain=config["domain"]), **base_args) + + streams = [ + LookerStream("color_collections", **args), + LookerStream("connections", **args), + ContentMetadata("content_metadata", **args), + ContentMetadata("content_metadata_access", **args), + Dashboards("dashboards", **args), + LookerStream("dashboard_elements", **args), + LookerStream("dashboard_filters", **args), + LookerStream("dashboard_layout_components", **args), + LookerStream("dashboard_layouts", **args), + LookerStream("datagroups", **args), + LookerStream("folders", **args), + LookerStream("folder_ancestors", **args), + LookerStream("git_branches", **args), + LookerStream("groups", **args), + LookerStream("homepage_items", **args), + LookerStream("homepage_sections", **args), + LookerStream("homepages", **args), + LookerStream("integration_hubs", **args), + LookerStream("integrations", **args), + LookerStream("legacy_features", **args), + Dashboards("lookml_dashboards", **args), + LookerStream("lookml_models", **args), + LookerStream("looks", **args), + LookerStream("model_sets", **args), + LookerStream("permission_sets", **args), + LookerStream("permissions", **args), + LookerStream("primary_homepage_sections", **args), + LookerStream("projects", **args), + LookerStream("project_files", **args), + QueryHistory(**base_args), + LookerStream("roles", **args), + LookerStream("role_groups", **args), + RunLooks(run_look_ids=config["run_look_ids"], **args) if config.get("run_look_ids") else None, + LookerStream("scheduled_plans", request_params={"all_users": "true"}, **args), + LookerStream("spaces", **args), + LookerStream("space_ancestors", **args), + LookerStream("user_attributes", **args), + LookerStream("user_attribute_group_values", **args), + LookerStream("user_attribute_values", request_params={"all_values": "true", "include_unset": "true"}, **args), + LookerStream("user_login_lockouts", **args), + LookerStream("user_sessions", **args), + LookerStream("users", **args), + LookerStream("versions", **args), + LookerStream("workspaces", **args), + ] + # stream RunLooks is dynamic and will be added if run_look_ids is not empty + # but we need to save streams' older + return [stream for stream in streams if stream] diff --git a/airbyte-integrations/connectors/source-looker/source_looker/streams.py b/airbyte-integrations/connectors/source-looker/source_looker/streams.py new file mode 100644 index 0000000000000..bc7163c71e9bc --- /dev/null +++ b/airbyte-integrations/connectors/source-looker/source_looker/streams.py @@ -0,0 +1,532 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import copy +import functools +from abc import ABC +from collections import defaultdict +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from prance import ResolvingParser + +API_VERSION = "3.1" + +CUSTOM_STREAM_NAMES = { + "/projects/{project_id}/files": "project_files", + "/roles/{role_id}/groups": "role_groups", + "/groups": "groups", + "/groups/{group_id}/groups": "group_groups", + "/user_attributes/{user_attribute_id}/group_values": "user_attribute_group_values", + "/users/{user_id}/attribute_values": "user_attribute_values", + "/users/{user_id}/sessions": "user_sessions", + "/dashboards": "dashboards", + "/roles": "roles", + "/users/{user_id}/roles": "user_roles", + "/spaces/{space_id}/dashboards": "space_dashboards", + "/folders/{folder_id}/dashboards": "folder_dashboards", + "/content_metadata/{content_metadata_id}": "content_metadata", + "/looks": "looks", + "/folders/{folder_id}/looks": "folder_looks", + "/spaces/{space_id}/looks": "space_looks", + "/looks/search": "search_looks", + "/looks/{look_id}": "look_info", + "/lookml_models/{lookml_model_name}/explores/{explore_name}": "explore_models", + "/dashboards/{dashboard_id}/dashboard_layouts": "dashboard_layouts", + "/folders/{folder_id}/ancestors": "folder_ancestors", + "/spaces/{space_id}/ancestors": "space_ancestors", + "/groups/{group_id}/users": "group_users", + "/users": "users", + "/roles/{role_id}/users": "role_users", +} + +FIELD_TYPE_MAPPING = { + "string": "string", + "date_date": "date", + "date_raw": "string", + "date": "date", + "date_week": "date", + "date_day_of_week": "string", + "date_day_of_week_index": "integer", + "date_month": "string", + "date_month_num": "integer", + "date_month_name": "string", + "date_day_of_month": "integer", + "date_fiscal_month_num": "integer", + "date_quarter": "string", + "date_quarter_of_year": "string", + "date_fiscal_quarter": "string", + "date_fiscal_quarter_of_year": "string", + "date_year": "integer", + "date_day_of_year": "integer", + "date_week_of_year": "integer", + "date_fiscal_year": "integer", + "date_time_of_day": "string", + "date_hour": "string", + "date_hour_of_day": "integer", + "date_minute": "string", + "date_second": "date-time", + "date_millisecond": "date-time", + "date_microsecond": "date-time", + "number": "number", + "int": "integer", + "list": "array", + "yesno": "boolean", +} + + +class LookerException(Exception): + pass + + +class BaseLookerStream(HttpStream, ABC): + """Base looker class""" + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + + @property + def primary_key(self) -> Optional[Union[str, List[str]]]: + return None + + def __init__(self, *, domain: str, **kwargs: Any): + self._domain = domain + super().__init__(**kwargs) + + @property + def authenticator(self) -> TokenAuthenticator: + if self._session.auth: + return self._session.auth + return super().authenticator + + @property + def url_base(self) -> str: + return f"https://{self._domain}/api/{API_VERSION}/" + + def next_page_token(self, response: requests.Response, **kwargs: Any) -> Optional[Mapping[str, Any]]: + return None + + +class SwaggerParser(BaseLookerStream): + """Convertor Swagger file to stream schemas + https:///api//swagger.json + """ + + class Endpoint: + def __init__(self, *, name: str, path: str, schema: Mapping[str, Any], operation_id: str, summary: str): + self.name, self.path, self.schema, self.operation_id, self.summary = name, path, schema, operation_id, summary + + def path(self, **kwargs: Any) -> str: + return "swagger.json" + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs: Any) -> Iterable[Mapping]: + yield ResolvingParser(spec_string=response.text) + + @functools.lru_cache(maxsize=None) + def get_endpoints(self) -> Mapping[str, "Endpoint"]: + parser = next(self.read_records(sync_mode=None)) + endpoints = {} + for path, methods in parser.specification["paths"].items(): + if not methods.get("get") or not methods["get"]["responses"].get("200"): + continue + get_data = methods["get"] + parts = path.split("/") + # self.logger.warning("dddddd %s" % path) + name = CUSTOM_STREAM_NAMES.get(path) + if not name: + name = "/".join(parts[-2:]) if parts[-1].endswith("}") else parts[-1] + if path == "/content_metadata_access": + path += "?content_metadata_id={content_metadata_id}" + + schema = get_data["responses"]["200"]["schema"] + endpoints[name] = self.Endpoint( + name=name, path=path, schema=self.format_schema(schema), summary=get_data["summary"], operation_id=get_data["operationId"] + ) + + # stream "lookml_dashboards" uses same endpoints + # "lookml_dashboards" and "dashboards" have one different only: + # "id" of "dashboards" is integer + # "id" of "lookml_dashboards" is string + dashboards_schema = endpoints["dashboards"].schema + lookml_dashboards_schema = copy.deepcopy(dashboards_schema) + dashboards_schema["items"]["properties"]["id"]["type"] = "integer" + lookml_dashboards_schema["items"]["properties"]["id"]["type"] = "string" + endpoints["lookml_dashboards"] = self.Endpoint( + name="lookml_dashboards", + path=endpoints["dashboards"].path, + schema=lookml_dashboards_schema, + summary=endpoints["dashboards"].summary, + operation_id=endpoints["dashboards"].operation_id, + ) + return endpoints + + @classmethod + def format_schema(cls, schema: Mapping[str, Any], key: str = None) -> Dict[str, Any]: + """Clean and validates all swagger "response" schemas + The Looker swagger file includes custom Locker fields (x-looker-...) and + it doesn't support multi typing( ["null", "..."]) + """ + updated_schema: Dict[str, Any] = {} + object_type: Union[str, List[str]] = schema.get("type") + if "properties" in schema: + object_type = ["null", "object"] + updated_sub_schemas: Dict[str, Any] = {} + for key, sub_schema in schema["properties"].items(): + updated_sub_schemas[key] = cls.format_schema(sub_schema, key=key) + updated_schema["properties"] = updated_sub_schemas + + elif "items" in schema: + object_type = ["null", "array"] + updated_schema["items"] = cls.format_schema(schema["items"]) + + if "format" in schema: + if schema["format"] == "int64" and (not key or not key.endswith("id")): + updated_schema["multipleOf"] = 10 ** -16 + object_type = "number" + else: + updated_schema["format"] = schema["format"] + if "description" in schema: + updated_schema["description"] = schema["description"] + + if schema.get("x-looker-nullable") is True and isinstance(object_type, str): + object_type = ["null", object_type] + updated_schema["type"] = object_type + return updated_schema + + +class LookerStream(BaseLookerStream, ABC): + # keys for correct mapping between parent and current streams. + # Several streams have some special aspects + parent_slice_key = "id" + custom_slice_key: str = None + + def __init__(self, name: str, swagger_parser: SwaggerParser, request_params: Mapping[str, Any] = None, **kwargs: Any): + self._swagger_parser = swagger_parser + self._name = name + self._request_params = request_params + super().__init__(**kwargs) + + @property + def endpoint(self) -> SwaggerParser.Endpoint: + """Extracts endpoint options""" + return self._swagger_parser.get_endpoints()[self._name] + + @property + def primary_key(self) -> Optional[Union[str, List[str]]]: + """not all streams have primary key""" + if self.get_json_schema()["properties"].get("id"): + return "id" + return None + + def generate_looker_stream(self, name: str, request_params: Mapping[str, Any] = None) -> "LookerStream": + """Generate a stream object. It can be used for loading of parent data""" + return LookerStream( + name, authenticator=self.authenticator, swagger_parser=self._swagger_parser, domain=self._domain, request_params=request_params + ) + + def get_parent_endpoints(self) -> List[SwaggerParser.Endpoint]: + parts = self.endpoint.path.split("/") + if len(parts) <= 3: + return [] + + parent_path = "/".join(parts[:-2]) + for endpoint in self._swagger_parser.get_endpoints().values(): + if endpoint.path == parent_path: + return [endpoint] + + # try to find a parent as the end of other path when a path has more then 1 parent + # e.g. /dashboard_layouts/{dashboard_layout_id}/dashboard_layout_components" + # => /dashboard_layouts => /dashboards/{dashboard_id}/dashboard_layouts + for endpoint in self._swagger_parser.get_endpoints().values(): + if endpoint.path.endswith(parent_path): + return [endpoint] + raise LookerException(f"not found the parent endpoint: {parent_path}") + + @property + def name(self) -> str: + return self._name + + def get_json_schema(self) -> Mapping[str, Any]: + # Overrides default logic. All schema should be generated dynamically. + schema = self.endpoint.schema.get("items") or self.endpoint.schema + return {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": schema["properties"]} + + def path(self, stream_slice: Mapping[str, Any], **kwargs: Any) -> str: + stream_slice = stream_slice or {} + return self.endpoint.path.format(**stream_slice)[1:] + + def request_params(self, **kwargs: Any) -> Optional[Mapping[str, Any]]: + return self._request_params or None + + def stream_slices(self, sync_mode: SyncMode, **kwargs: Any) -> Iterable[Optional[Mapping[str, Any]]]: + parent_endpoints = self.get_parent_endpoints() + if not parent_endpoints: + yield None + return + for parent_endpoint in parent_endpoints: + parent_stream = self.generate_looker_stream(parent_endpoint.name) + + # if self.custom_slice_key is None, this logic will generate it itself with the following rule: + # parent_name has the 's' at the end and its template key has "_id" at the end + # e.g. dashboards => dashboard_id + parent_key = self.custom_slice_key or parent_endpoint.name[:-1] + "_id" + for slice in parent_stream.stream_slices(sync_mode=sync_mode): + for item in parent_stream.read_records(sync_mode=sync_mode, stream_slice=slice): + if item[self.parent_slice_key]: + yield {parent_key: item[self.parent_slice_key]} + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs: Any) -> Iterable[Mapping]: + """Parses data. The Looker API doesn't support pagination logis. + Thus all responses are or JSON list or target single object. + """ + data = response.json() + if isinstance(data, list): + yield from data + else: + yield data + + +class ContentMetadata(LookerStream): + """ContentMetadata stream has personal customization. Because it has several parent streams""" + + parent_slice_key = "content_metadata_id" + custom_slice_key = "content_metadata_id" + + def get_parent_endpoints(self) -> List[SwaggerParser.Endpoint]: + parent_names = ("dashboards", "folders", "homepages", "looks", "spaces") + return [endpoint for name, endpoint in self._swagger_parser.get_endpoints().items() if name in parent_names] + + +class QueryHistory(BaseLookerStream): + """This stream is custom Looker query. That its response has a individual static schema.""" + + http_method = "POST" + cursor_field = "history_created_time" + # all connector's request should have this value of as prefix of queries' client_id + airbyte_client_id_prefix = "AiRbYtE2" + + @property + def primary_key(self) -> Optional[Union[str, List[str]]]: + return ["query_id", "history_created_time"] + + @property + def state_checkpoint_interval(self) -> Optional[int]: + """this is a workaround: the Airbyte CDK forces for save the latest state after reading of all records""" + if self._is_finished: + return 1 + return 100 + + def path(self, **kwargs: Any) -> str: + return "queries/run/json" + + def __init__(self, **kwargs: Any): + super().__init__(**kwargs) + self._last_query_id = None + self._is_finished = False + + def get_query_client_id(self, stream_state: MutableMapping[str, Any]) -> str: + """ + The query client_id is used for filtering because this query metadata is added to a response of this request + and the connector should skip our request information. + Values of the client_id is unique for every request body. it must be changed if any query symbol is changed. + But for incremental logic we have to add dynamic filter conditions. The single query's updating is stream_state value + thus it can be used as a part of client_id values. e.g.: + stream_state 2050-01-01T00:00:00Z -> client_id AiRbYtE225246479800000 + """ + latest_created_time = (stream_state or {}).get(self.cursor_field) + timestamp = 0 + if latest_created_time: + dt = pendulum.parse(latest_created_time) # type: ignore[attr-defined] + timestamp = int(dt.timestamp()) + # client_id has the hard-set length (22 symbols) this we add "0" to the end + return f"{self.airbyte_client_id_prefix}{timestamp}".ljust(22, "0") + + def request_body_json(self, stream_state: MutableMapping[str, Any], **kwargs: Any) -> Optional[Mapping]: + latest_created_time = (stream_state or {}).get(self.cursor_field) + + if not latest_created_time: + latest_created_time = "1970-01-01T00:00:00Z" + + dt = pendulum.parse(latest_created_time) # type: ignore[attr-defined] + dt_func = f"date_time({dt.year}, {dt.month}, {dt.day}, {dt.hour}, {dt.minute}, {dt.second})" + client_id = self.get_query_client_id(stream_state) + + # need to add the custom client_id value. It is used for filtering + # its value shouldn't be changed in the future + return { + "model": "i__looker", + "view": "history", + "limit": "10000", + "client_id": client_id, + "fields": [ + "query.id", + "history.created_date", + "history.created_time", + "query.client_id", + "query.model", + "query.view", + "space.id", + "look.id", + "dashboard.id", + "user.id", + "history.query_run_count", + "history.total_runtime", + ], + "filters": { + "query.model": "-EMPTY", + "history.runtime": "NOT NULL", + "user.is_looker": "No", + }, + "filter_expression": f"${{history.created_time}} > {dt_func}", + "sorts": [ + "history.created_time", + "query.id", + ], + } + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs: Any) -> Iterable[Mapping]: + records = response.json() + for i in range(len(records)): + record = records[i] + if record.get("looker_error"): + raise LookerException(f"Locker Error: {record['looker_error']}") + if (record.get("query.client_id") or "").startswith(self.airbyte_client_id_prefix): + # skip all native connector's requests + continue + # query.column_limit is used for filtering only + record.pop("query.client_id", None) + + # convert date to ISO format: 2021-10-12 10:46 => 2021-10-12T10:46:00Z + record[self.cursor_field] = record.pop("history.created_time").replace(" ", "T") + "Z" + + if i >= len(records) - 1: + self._is_finished = True + # convert history.created_date => history_created_date etc + yield {k.replace(".", "_"): v for k, v in record.items()} + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + record_query_id = latest_record["query_id"] + if not self._is_finished and self._last_query_id == record_query_id: + if not self._last_query_id: + self._last_query_id = record_query_id + return current_stream_state + return {self.cursor_field: max((current_stream_state or {}).get(self.cursor_field) or "", latest_record[self.cursor_field])} + + +class RunLooks(LookerStream): + """ " + Runs ready looks' requests + Docs: https://docs.looker.com/reference/api-and-integration/api-reference/v4.0/look#run_look + """ + + @property + def primary_key(self) -> Optional[Union[str, List[str]]]: + return None + + def __init__(self, run_look_ids: List[str], **kwargs: Any): + self._run_look_ids = run_look_ids + super().__init__(name="run_looks", **kwargs) + + @staticmethod + def _get_run_look_key(look: Mapping[str, Any]) -> str: + return f"{look['id']} - {look['title']}" + + def path(self, stream_slice: Mapping[str, Any], **kwargs: Any) -> str: + return f'looks/{stream_slice["id"]}/run/json' + + def stream_slices(self, sync_mode: SyncMode, **kwargs: Any) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream = self.generate_looker_stream( + "search_looks", request_params={"id": ",".join(self._run_look_ids), "limit": "10000", "fields": "id,title,model(id)"} + ) + found_look_ids = [] + for slice in parent_stream.stream_slices(sync_mode=sync_mode): + for item in parent_stream.read_records(sync_mode=sync_mode, stream_slice=slice): + if isinstance(item["model"], dict): + item["model"] = item.pop("model")["id"] + found_look_ids.append(item["id"]) + yield item + diff_ids = set(self._run_look_ids) - set(str(id) for id in found_look_ids) + if diff_ids: + raise LookerException(f"not found run_look_ids: {diff_ids}") + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs: Any) -> Iterable[Mapping]: + for record in super().parse_response(response=response, stream_slice=stream_slice, **kwargs): + yield {self._get_run_look_key(stream_slice): {k.replace(".", "_"): v for k, v in record.items()}} + + def get_json_schema(self) -> Mapping[str, Any]: + """ + For a given LookML model and field, looks up its type and generates + its properties for the run_look endpoint JSON Schema + """ + properties = {} + for look_info in self.stream_slices(sync_mode=None): + look_properties = {} + for explore, fields in self._get_look_fields(look_info["id"]).items(): + explore_fields = self._get_explore_field_types(look_info["model"], explore) + look_properties.update({field.replace(".", "_"): explore_fields[field] for field in fields}) + properties[self._get_run_look_key(look_info)] = { + "title": look_info["title"], + "properties": look_properties, + "type": ["null", "object"], + "additionalProperties": False, + } + # raise LookerException(properties) + return { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "type": "object", + "properties": properties, + } + + def _get_look_fields(self, look_id: int) -> Mapping[str, List[str]]: + stream = self.generate_looker_stream("look_info", request_params={"fields": "query(fields)"}) + slice = {"look_id": look_id} + for item in stream.read_records(sync_mode=None, stream_slice=slice): + explores = defaultdict(list) + for field in item["query"]["fields"]: + explores[field.split(".")[0]].append(field) + return explores + + raise LookerException(f"not found fields for the look ID: {look_id}") + + def _get_explore_field_types(self, model: str, explore: str) -> Mapping[str, Any]: + """ + For a given LookML model and explore, looks up its dimensions/measures + and their types for run_look endpoint JSON Schema generation + """ + stream = self.generate_looker_stream( + "explore_models", request_params={"fields": "fields(dimensions(name, type),measures(name, type))"} + ) + slice = {"lookml_model_name": model, "explore_name": explore} + data = next(stream.read_records(sync_mode=None, stream_slice=slice))["fields"] + fields = {} + for dimension in data["dimensions"]: + fields[dimension["name"]] = FIELD_TYPE_MAPPING.get(dimension["type"]) or "string" + for measure in data["measures"]: + fields[measure["name"]] = FIELD_TYPE_MAPPING.get(measure["type"]) or "number" + field_types = {} + for field_name in fields: + if "date" in fields[field_name]: + schema = {"type": ["null", "string"], "format": fields[field_name]} + else: + schema = {"type": ["null", fields[field_name]]} + field_types[field_name] = schema + return field_types + + +class Dashboards(LookerStream): + """Customization for dashboards stream because for 2 diff stream there is single endpoint only""" + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs: Any) -> Iterable[Mapping]: + for record in super().parse_response(response=response, stream_slice=stream_slice, **kwargs): + # "id" of "dashboards" is integer + # "id" of "lookml_dashboards" is string + if self._name == "dashboards" and isinstance(record["id"], int): + yield record + elif self._name == "lookml_dashboards" and isinstance(record["id"], str): + yield record diff --git a/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py index e1814314fc3b0..2dc6291301bf7 100644 --- a/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py @@ -3,5 +3,6 @@ # -def test_example_method(): +def test_example_method() -> None: assert True + return diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java index 3b73693008ef7..39e7a2cadfe38 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java @@ -26,7 +26,6 @@ import io.airbyte.protocol.models.SyncMode; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; import java.util.HashMap; import java.util.List; import org.bson.BsonArray; @@ -137,11 +136,6 @@ protected JsonNode getState() throws Exception { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() throws Exception { - return Collections.emptyList(); - } - @Test void testSpec() throws Exception { final ConnectorSpecification actual = new MongodbSourceStrictEncrypt().spec(); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java index 965d1749922da..c093c970a65c6 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java @@ -18,7 +18,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -72,9 +71,4 @@ protected JsonNode getState() throws Exception { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() throws Exception { - return Collections.emptyList(); - } - } diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java index 6718ce2fd2b8d..2e0881cfc5501 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java @@ -20,9 +20,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.sql.SQLException; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.apache.commons.lang3.RandomStringUtils; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.utility.DockerImageName; @@ -114,9 +112,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - } diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 5691347a99a61..5bde73f85b611 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.13 +LABEL io.airbyte.version=0.3.14 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java index 5168e31668887..060dbeaa6bb97 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java @@ -23,7 +23,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Objects; @@ -149,11 +148,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java index 8bce8d4aa0720..b162b78534d77 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java @@ -21,7 +21,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.List; import org.testcontainers.containers.MSSQLServerContainer; @@ -87,11 +86,6 @@ protected JsonNode getState() { return null; } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws InterruptedException { container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java index ab31da0972538..fb9c59dddbd44 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java @@ -331,7 +331,7 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'", "null") - .addExpectedValues("0001-01-01", "9999-12-31", "1999-01-08", null) + .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", "1999-01-08T00:00:00Z", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java index 78302dc24ccae..c2597b80456ea 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java @@ -20,9 +20,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.sql.SQLException; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.testcontainers.containers.MSSQLServerContainer; public class MssqlSourceAcceptanceTest extends SourceAcceptanceTest { @@ -97,11 +95,6 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - private static Database getDatabase(final JsonNode config) { return Databases.createDatabase( config.get("username").asText(), diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java index 9535884de4d9d..9148e40878eff 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java @@ -25,9 +25,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -118,11 +116,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index 9c811ed82bf64..1aac8a064b23e 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -43,7 +43,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 5 diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index 050cf00ec6d05..c6a652f3e48ef 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -43,7 +43,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 5 diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractSshMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractSshMySqlSourceAcceptanceTest.java index cfc7476ed9de0..cba86252ae774 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractSshMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractSshMySqlSourceAcceptanceTest.java @@ -20,9 +20,7 @@ import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.nio.file.Path; -import java.util.Collections; import java.util.HashMap; -import java.util.List; public abstract class AbstractSshMySqlSourceAcceptanceTest extends SourceAcceptanceTest { @@ -81,11 +79,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java index caf841607ab1f..3bd6b24367462 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java @@ -22,7 +22,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.List; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -83,11 +82,6 @@ protected JsonNode getState() { return null; } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) { container = new MySQLContainer<>("mysql:8.0"); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java index 77087397ffb3d..9cc1cf1cc6bbc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java @@ -22,9 +22,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -114,11 +112,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java index 44c381069cc9a..656436bc72a9f 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java @@ -21,7 +21,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -121,11 +120,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java index cd13e75b49582..276da38c96fe5 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java @@ -16,7 +16,6 @@ import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.*; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Objects; @@ -138,11 +137,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java index 286760b44cf85..94b1d0f1a473d 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java @@ -20,7 +20,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -117,11 +116,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-persistiq/setup.py b/airbyte-integrations/connectors/source-persistiq/setup.py index af1420f9687cc..cdcac84c0c076 100644 --- a/airbyte-integrations/connectors/source-persistiq/setup.py +++ b/airbyte-integrations/connectors/source-persistiq/setup.py @@ -11,7 +11,7 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", - "pytest-mock~=3.6.1", + "pytest-mock~=3.6.1", "requests_mock==1.8.0", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java index 8729d20ab487b..b48d48b0eff7b 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java @@ -22,9 +22,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; @@ -118,11 +116,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 98017bc25cc9b..6a0fb1261224b 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.2 +LABEL io.airbyte.version=0.4.4 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index bf8d5c02acada..61b7bf25dda8b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Postgres Source Spec", @@ -85,7 +85,7 @@ { "title": "Logical Replication (CDC)", "additionalProperties": false, - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the Postgres Source docs for more information.", + "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the Postgres Source docs for more information.", "required": ["method", "replication_slot", "publication"], "properties": { "method": { @@ -97,18 +97,21 @@ }, "plugin": { "type": "string", - "description": "A logical decoding plug-in installed on the PostgreSQL server. `pgoutput` plug-in is used by default.\nIf replication table contains a lot of big jsonb values it is recommended to use `wal2json` plug-in. For more information about `wal2json` plug-in read Postgres Source docs.", + "title": "Plugin", + "description": "A logical decoding plug-in installed on the PostgreSQL server. `pgoutput` plug-in is used by default.\nIf replication table contains a lot of big jsonb values it is recommended to use `wal2json` plug-in. For more information about `wal2json` plug-in read Postgres Source docs.", "enum": ["pgoutput", "wal2json"], "default": "pgoutput", "order": 1 }, "replication_slot": { "type": "string", + "title": "Replication Slot", "description": "A plug-in logical replication slot.", "order": 2 }, "publication": { "type": "string", + "title": "Publication", "description": "A Postgres publication used for consuming changes.", "order": 3 } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java index 081b8928cf88a..aac2caa007c88 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java @@ -22,7 +22,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; import java.util.List; import org.jooq.SQLDialect; @@ -123,11 +122,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index ed8c94b8b51f6..35f4b21675352 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -21,7 +21,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; import java.util.List; import org.jooq.SQLDialect; @@ -153,11 +152,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java index c4964892eac64..5543cc40879b9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java @@ -225,7 +225,8 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("'January 7, 1999'", "'1999-01-08'", "'1/9/1999'", "'January 10, 99 BC'", "'January 11, 99 AD'", "null") - .addExpectedValues("1999-01-07", "1999-01-08", "1999-01-09", "0099-01-10", "1999-01-11", null) + .addExpectedValues("1999-01-07T00:00:00Z", "1999-01-08T00:00:00Z", "1999-01-09T00:00:00Z", "0099-01-10T00:00:00Z", "1999-01-11T00:00:00Z", + null) .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index 184215bc93c45..b275dcdf091b3 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -21,7 +21,6 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; -import java.util.Collections; import java.util.HashMap; import java.util.List; import org.jooq.SQLDialect; @@ -128,11 +127,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile index b231f28aafda3..0d17906bba651 100644 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ b/airbyte-integrations/connectors/source-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.6 +LABEL io.airbyte.version=0.3.7 LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java index f1196d9e5e2f5..b8470c82146e3 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java @@ -7,10 +7,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.relationaldb.TableInfo; +import io.airbyte.protocol.models.CommonField; import java.sql.JDBCType; import java.util.ArrayList; import java.util.List; @@ -22,6 +25,8 @@ public class RedshiftSource extends AbstractJdbcSource implements Sour private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSource.class); public static final String DRIVER_CLASS = "com.amazon.redshift.jdbc.Driver"; + private static final String SCHEMAS = "schemas"; + private List schemas; // todo (cgardens) - clean up passing the dialect as null versus explicitly adding the case to the // constructor. @@ -39,7 +44,20 @@ public JsonNode toDatabaseConfig(final JsonNode redshiftConfig) { redshiftConfig.get("host").asText(), redshiftConfig.get("port").asText(), redshiftConfig.get("database").asText())); + + if (redshiftConfig.has(SCHEMAS) && redshiftConfig.get(SCHEMAS).isArray()) { + schemas = new ArrayList<>(); + for (final JsonNode schema : redshiftConfig.get(SCHEMAS)) { + schemas.add(schema.asText()); + } + + if (schemas != null && !schemas.isEmpty()) { + additionalProperties.add("currentSchema=" + String.join(",", schemas)); + } + } + addSsl(additionalProperties); + builder.put("connection_properties", String.join(";", additionalProperties)); return Jsons.jsonNode(builder @@ -51,6 +69,25 @@ private void addSsl(final List additionalProperties) { additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); } + @Override + public List>> discoverInternal(JdbcDatabase database) throws Exception { + if (schemas != null && !schemas.isEmpty()) { + // process explicitly selected (from UI) schemas + final List>> internals = new ArrayList<>(); + for (String schema : schemas) { + LOGGER.debug("Discovering schema: {}", schema); + internals.addAll(super.discoverInternal(database, schema)); + } + for (TableInfo> info : internals) { + LOGGER.debug("Found table (schema: {}): {}", info.getNameSpace(), info.getName()); + } + return internals; + } else { + LOGGER.info("No schemas explicitly set on UI to process, so will process all of existing schemas in DB"); + return super.discoverInternal(database); + } + } + @Override public Set getExcludedInternalNameSpaces() { return Set.of("information_schema", "pg_catalog", "pg_internal", "catalog_history"); diff --git a/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json index 9d0b5888d3590..a8dbe721a88ca 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json @@ -10,7 +10,8 @@ "host": { "title": "Host", "description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com).", - "type": "string" + "type": "string", + "order": 1 }, "port": { "title": "Port", @@ -19,24 +20,40 @@ "minimum": 0, "maximum": 65536, "default": 5439, - "examples": ["5439"] + "examples": ["5439"], + "order": 2 }, "database": { "title": "Database", "description": "Name of the database.", "type": "string", - "examples": ["master"] + "examples": ["master"], + "order": 3 + }, + "schemas": { + "title": "Schemas", + "description": "The list of schemas to sync from. Specify one or more explicitly or keep empty to process all schemas. Schema names are case sensitive.", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 0, + "uniqueItems": true, + "examples": ["public"], + "order": 4 }, "username": { "title": "Username", "description": "Username to use to access the database.", - "type": "string" + "type": "string", + "order": 5 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", - "airbyte_secret": true + "airbyte_secret": true, + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java index f65fd4ea75d00..4ef06497b823d 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java @@ -4,7 +4,10 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; @@ -15,6 +18,8 @@ import io.airbyte.integrations.source.redshift.RedshiftSource; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; @@ -22,27 +27,49 @@ import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.nio.file.Path; import java.sql.SQLException; -import java.util.Collections; import java.util.HashMap; import java.util.List; public class RedshiftSourceAcceptanceTest extends SourceAcceptanceTest { + protected static final List FIELDS = List.of( + Field.of("c_custkey", JsonSchemaPrimitive.NUMBER), + Field.of("c_name", JsonSchemaPrimitive.STRING), + Field.of("c_nation", JsonSchemaPrimitive.STRING)); + // This test case expects an active redshift cluster that is useable from outside of vpc - protected JsonNode config; + protected ObjectNode config; protected JdbcDatabase database; protected String schemaName; + protected String schemaToIgnore; protected String streamName; - protected static JsonNode getStaticConfig() { - return Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); + protected static ObjectNode getStaticConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); } @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { config = getStaticConfig(); - database = Databases.createJdbcDatabase( + database = createDatabase(config); + + schemaName = Strings.addRandomSuffix("integration_test", "_", 5).toLowerCase(); + schemaToIgnore = schemaName + "shouldIgnore"; + + // limit the connection to one schema only + config = config.set("schemas", Jsons.jsonNode(List.of(schemaName))); + + // create a test data + createTestData(database, schemaName); + + // create a schema with data that will not be used for testing, but would be used to check schema + // filtering. This one should not be visible in results + createTestData(database, schemaToIgnore); + } + + protected static JdbcDatabase createDatabase(final JsonNode config) { + return Databases.createJdbcDatabase( config.get("username").asText(), config.get("password").asText(), String.format("jdbc:redshift://%s:%s/%s", @@ -50,8 +77,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get("port").asText(), config.get("database").asText()), RedshiftSource.DRIVER_CLASS); + } - schemaName = Strings.addRandomSuffix("integration_test", "_", 5).toLowerCase(); + protected void createTestData(final JdbcDatabase database, final String schemaName) + throws SQLException { final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); database.execute(connection -> { connection.createStatement().execute(createSchemaQuery); @@ -60,12 +89,15 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc streamName = "customer"; final String fqTableName = JdbcUtils.getFullyQualifiedTableName(schemaName, streamName); final String createTestTable = - String.format("CREATE TABLE IF NOT EXISTS %s (c_custkey INTEGER, c_name VARCHAR(16), c_nation VARCHAR(16));\n", fqTableName); + String.format( + "CREATE TABLE IF NOT EXISTS %s (c_custkey INTEGER, c_name VARCHAR(16), c_nation VARCHAR(16));\n", + fqTableName); database.execute(connection -> { connection.createStatement().execute(createTestTable); }); - final String insertTestData = String.format("insert into %s values (1, 'Chris', 'France');\n", fqTableName); + final String insertTestData = String.format("insert into %s values (1, 'Chris', 'France');\n", + fqTableName); database.execute(connection -> { connection.createStatement().execute(insertTestData); }); @@ -73,8 +105,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc @Override protected void tearDown(final TestDestinationEnv testEnv) throws SQLException { - final String dropSchemaQuery = String.format("DROP SCHEMA IF EXISTS %s CASCADE", schemaName); - database.execute(connection -> connection.createStatement().execute(dropSchemaQuery)); + database.execute(connection -> connection.createStatement() + .execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schemaName))); + database.execute(connection -> connection.createStatement() + .execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schemaToIgnore))); } @Override @@ -94,22 +128,24 @@ protected JsonNode getConfig() { @Override protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return CatalogHelpers.createConfiguredAirbyteCatalog( - streamName, - schemaName, - Field.of("c_custkey", JsonSchemaPrimitive.NUMBER), - Field.of("c_name", JsonSchemaPrimitive.STRING), - Field.of("c_nation", JsonSchemaPrimitive.STRING)); + return CatalogHelpers.createConfiguredAirbyteCatalog(streamName, schemaName, FIELDS); } @Override - protected List getRegexTests() { - return Collections.emptyList(); + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); } @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); + protected void verifyCatalog(final AirbyteCatalog catalog) { + final List streams = catalog.getStreams(); + // only one stream is expected; the schema that should be ignored + // must not be included in the retrieved catalog + assertEquals(1, streams.size()); + final AirbyteStream actualStream = streams.get(0); + assertEquals(schemaName, actualStream.getNamespace()); + assertEquals(streamName, actualStream.getName()); + assertEquals(CatalogHelpers.fieldsToJsonSchema(FIELDS), actualStream.getJsonSchema()); } } diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java index a6b9316651d69..144a78bf1d364 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java @@ -4,19 +4,15 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import io.airbyte.commons.string.Strings; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.db.Databases; -import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.redshift.RedshiftSource; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; public class RedshiftSslSourceAcceptanceTest extends RedshiftSourceAcceptanceTest { - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - config = getStaticConfig(); - - database = Databases.createJdbcDatabase( + protected static JdbcDatabase createDatabase(final JsonNode config) { + return Databases.createJdbcDatabase( config.get("username").asText(), config.get("password").asText(), String.format("jdbc:redshift://%s:%s/%s", @@ -26,25 +22,6 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc RedshiftSource.DRIVER_CLASS, "ssl=true;" + "sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); - - schemaName = Strings.addRandomSuffix("integration_test", "_", 5).toLowerCase(); - final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); - database.execute(connection -> { - connection.createStatement().execute(createSchemaQuery); - }); - - streamName = "customer"; - final String fqTableName = JdbcUtils.getFullyQualifiedTableName(schemaName, streamName); - final String createTestTable = - String.format("CREATE TABLE IF NOT EXISTS %s (c_custkey INTEGER, c_name VARCHAR(16), c_nation VARCHAR(16));\n", fqTableName); - database.execute(connection -> { - connection.createStatement().execute(createTestTable); - }); - - final String insertTestData = String.format("insert into %s values (1, 'Chris', 'France');\n", fqTableName); - database.execute(connection -> { - connection.createStatement().execute(insertTestData); - }); } } diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index 13a78f12d7def..3b9f1ddc2fa05 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -25,5 +25,5 @@ COPY source_salesforce ./source_salesforce ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.20 +LABEL io.airbyte.version=0.1.21 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py index 88c19292e5d15..766fd90c4d4cb 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py @@ -34,7 +34,7 @@ def should_give_up(exc): if exc.response is not None and exc.response.status_code == codes.forbidden: error_data = exc.response.json()[0] if error_data.get("errorCode", "") == "REQUEST_LIMIT_EXCEEDED": - give_up = False + give_up = True if give_up: logger.info(f"Giving up for returned HTTP status: {exc.response.status_code}, body: {exc.response.text}") diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py index 1784d27407e32..e7edfb658f6c9 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from airbyte_cdk.sources.utils.schema_helpers import split_config +from requests import codes, exceptions from .api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .streams import BulkIncrementalSalesforceStream, BulkSalesforceStream, IncrementalSalesforceStream, SalesforceStream @@ -24,12 +25,24 @@ def _get_sf_object(config: Mapping[str, Any]) -> Salesforce: return sf def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: - _ = self._get_sf_object(config) - return True, None + try: + _ = self._get_sf_object(config) + return True, None + except exceptions.HTTPError as error: + error_data = error.response.json()[0] + error_code = error_data.get("errorCode") + if error.response.status_code == codes.FORBIDDEN and error_code == "REQUEST_LIMIT_EXCEEDED": + logger.warn(f"API Call limit is exceeded. Error message: '{error_data.get('message')}'") + return False, "API Call limit is exceeded" @classmethod def generate_streams( - cls, config: Mapping[str, Any], stream_names: List[str], sf_object: Salesforce, state: Mapping[str, Any] = None, stream_objects: List = None + cls, + config: Mapping[str, Any], + stream_names: List[str], + sf_object: Salesforce, + state: Mapping[str, Any] = None, + stream_objects: List = None, ) -> List[Stream]: """ "Generates a list of stream by their names. It can be used for different tests too""" authenticator = TokenAuthenticator(sf_object.access_token) @@ -96,6 +109,14 @@ def read( connector_state=connector_state, internal_config=internal_config, ) + except exceptions.HTTPError as error: + error_data = error.response.json()[0] + error_code = error_data.get("errorCode") + if error.response.status_code == codes.FORBIDDEN and error_code == "REQUEST_LIMIT_EXCEEDED": + logger.warn(f"API Call limit is exceeded. Error message: '{error_data.get('message')}'") + break # if got 403 rate limit response, finish the sync with success. + raise error + except Exception as e: logger.exception(f"Encountered an exception while reading stream {self.name}") raise e diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/configured_catalog.json b/airbyte-integrations/connectors/source-salesforce/unit_tests/configured_catalog.json new file mode 100644 index 0000000000000..647831c8b86f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/configured_catalog.json @@ -0,0 +1,28 @@ +{ + "streams": [ + { + "stream": { + "name": "Account", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["LastModifiedDate"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "Asset", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["SystemModstamp"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/unit_test.py index 85ba0850eed3d..b70b9ddbf1305 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/unit_test.py @@ -4,17 +4,32 @@ import csv import io +import json from unittest.mock import Mock import pytest import requests_mock -from airbyte_cdk.models import SyncMode +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type from requests.exceptions import HTTPError from source_salesforce.api import Salesforce from source_salesforce.source import SourceSalesforce from source_salesforce.streams import BulkIncrementalSalesforceStream, BulkSalesforceStream, IncrementalSalesforceStream, SalesforceStream +@pytest.fixture(scope="module") +def configured_catalog(): + with open("unit_tests/configured_catalog.json") as f: + data = json.loads(f.read()) + return ConfiguredAirbyteCatalog.parse_obj(data) + + +@pytest.fixture(scope="module") +def state(): + state = {"Account": {"LastModifiedDate": "2021-10-01T21:18:20.000Z"}, "Asset": {"SystemModstamp": "2021-10-02T05:08:29.000Z"}} + return state + + @pytest.fixture(scope="module") def stream_config(): """Generates streams settings for BULK logic""" @@ -319,6 +334,151 @@ def test_discover_with_streams_criteria_param(streams_criteria, predicted_filter assert sorted(filtered_streams) == sorted(predicted_filtered_streams) +def test_check_connection_rate_limit(stream_config): + source = SourceSalesforce() + logger = AirbyteLogger() + + json_response = [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "TotalRequests Limit exceeded."}] + url = "https://login.salesforce.com/services/oauth2/token" + with requests_mock.Mocker() as m: + m.register_uri("POST", url, json=json_response, status_code=403) + result, msg = source.check_connection(logger, stream_config) + assert result is False + assert msg == "API Call limit is exceeded" + + +def configure_request_params_mock(stream_1, stream_2): + stream_1.request_params = Mock() + stream_1.request_params.return_value = {"q": "query"} + + stream_2.request_params = Mock() + stream_2.request_params.return_value = {"q": "query"} + + +def test_rate_limit_bulk(stream_config, stream_api, configured_catalog, state): + """ + Connector should stop the sync if one stream reached rate limit + stream_1, stream_2, stream_3, ... + While reading `stream_1` if 403 (Rate Limit) is received, it should finish that stream with success and stop the sync process. + Next streams should not be executed. + """ + stream_1: BulkIncrementalSalesforceStream = _generate_stream("Account", stream_config, stream_api) + stream_2: BulkIncrementalSalesforceStream = _generate_stream("Asset", stream_config, stream_api) + streams = [stream_1, stream_2] + configure_request_params_mock(stream_1, stream_2) + + stream_1.page_size = 6 + stream_1.state_checkpoint_interval = 5 + + source = SourceSalesforce() + source.streams = Mock() + source.streams.return_value = streams + logger = AirbyteLogger() + + json_response = [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "TotalRequests Limit exceeded."}] + with requests_mock.Mocker() as m: + for stream in streams: + creation_responses = [] + for page in [1, 2]: + job_id = f"fake_job_{page}_{stream.name}" + creation_responses.append({"json": {"id": job_id}}) + + m.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"}) + + resp = ["Field1,LastModifiedDate,ID"] + [f"test,2021-11-0{i},{i}" for i in range(1, 7)] # 6 records per page + + if page == 1: + # Read the first page successfully + m.register_uri("GET", stream.path() + f"/{job_id}/results", text="\n".join(resp)) + else: + # Requesting for results when reading second page should fail with 403 (Rate Limit error) + m.register_uri("GET", stream.path() + f"/{job_id}/results", status_code=403, json=json_response) + + m.register_uri("DELETE", stream.path() + f"/{job_id}") + + m.register_uri("POST", stream.path(), creation_responses) + + result = [i for i in source.read(logger=logger, config=stream_config, catalog=configured_catalog, state=state)] + assert stream_1.request_params.called + assert ( + not stream_2.request_params.called + ), "The second stream should not be executed, because the first stream finished with Rate Limit." + + records = [item for item in result if item.type == Type.RECORD] + assert len(records) == 6 # stream page size: 6 + + state_record = [item for item in result if item.type == Type.STATE][0] + assert state_record.state.data["Account"]["LastModifiedDate"] == "2021-11-05" # state checkpoint interval is 5. + + +def test_rate_limit_rest(stream_config, stream_api, configured_catalog, state): + """ + Connector should stop the sync if one stream reached rate limit + stream_1, stream_2, stream_3, ... + While reading `stream_1` if 403 (Rate Limit) is received, it should finish that stream with success and stop the sync process. + Next streams should not be executed. + """ + + stream_1: IncrementalSalesforceStream = _generate_stream("Account", stream_config, stream_api, state=state) + stream_2: IncrementalSalesforceStream = _generate_stream("Asset", stream_config, stream_api, state=state) + + stream_1.state_checkpoint_interval = 3 + configure_request_params_mock(stream_1, stream_2) + + source = SourceSalesforce() + source.streams = Mock() + source.streams.return_value = [stream_1, stream_2] + + logger = AirbyteLogger() + + next_page_url = "/services/data/v52.0/query/012345" + response_1 = { + "done": False, + "totalSize": 10, + "nextRecordsUrl": next_page_url, + "records": [ + { + "ID": 1, + "LastModifiedDate": "2021-11-15", + }, + { + "ID": 2, + "LastModifiedDate": "2021-11-16", + }, + { + "ID": 3, + "LastModifiedDate": "2021-11-17", # check point interval + }, + { + "ID": 4, + "LastModifiedDate": "2021-11-18", + }, + { + "ID": 5, + "LastModifiedDate": "2021-11-19", + }, + ], + } + response_2 = [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "TotalRequests Limit exceeded."}] + + with requests_mock.Mocker() as m: + m.register_uri("GET", stream_1.path(), json=response_1, status_code=200) + m.register_uri("GET", next_page_url, json=response_2, status_code=403) + + result = [i for i in source.read(logger=logger, config=stream_config, catalog=configured_catalog, state=state)] + + assert stream_1.request_params.called + assert ( + not stream_2.request_params.called + ), "The second stream should not be executed, because the first stream finished with Rate Limit." + + records = [item for item in result if item.type == Type.RECORD] + assert len(records) == 5 + + state_record = [item for item in result if item.type == Type.STATE][0] + assert state_record.state.data["Account"]["LastModifiedDate"] == "2021-11-17" + + def test_discover_only_queryable(stream_config): sf_object = Salesforce(**stream_config) sf_object.login = Mock() diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json index 62de3fc0db353..fb75d79434a2d 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json @@ -38,7 +38,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", "type": "string", "order": 5 }, diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java index 9d15d370ef5aa..dafbd2962751a 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java @@ -11,9 +11,7 @@ import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; -import java.util.Collections; import java.util.HashMap; -import java.util.List; public class ScaffoldJavaJdbcSourceAcceptanceTest extends SourceAcceptanceTest { @@ -55,11 +53,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { return null; } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - @Override protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json index f5ade7534ef79..b7ef5f06b5748 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json @@ -13,7 +13,7 @@ "multipass_identifier": { "type": ["null", "string"] }, - "shop_url":{ + "shop_url": { "type": ["null", "string"] }, "default_address": { diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index db54e5e234328..8103c7d1e71dd 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -66,13 +66,13 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp if isinstance(records, dict): # for cases when we have a single record as dict # add shop_url to the record to make querying easy - records['shop_url'] = self.config["shop"] + records["shop_url"] = self.config["shop"] yield self._transformer.transform(records) else: # for other cases for record in records: # add shop_url to the record to make querying easy - record['shop_url'] = self.config["shop"] + record["shop_url"] = self.config["shop"] yield self._transformer.transform(record) @property diff --git a/airbyte-integrations/connectors/source-slack/Dockerfile b/airbyte-integrations/connectors/source-slack/Dockerfile index e76c29e9c5306..190c716155433 100644 --- a/airbyte-integrations/connectors/source-slack/Dockerfile +++ b/airbyte-integrations/connectors/source-slack/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-slack diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json index 5b94bf0c604da..e616ac54b5659 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json @@ -38,8 +38,7 @@ "type": ["null", "string"] }, "updated": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 22356354f9002..3c35d62e1d0ea 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index cdffb1abac6f1..b404e4fc3b3b3 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -33,23 +33,34 @@ public static void main(final String[] args) throws Exception { @Override public JsonNode toDatabaseConfig(final JsonNode config) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("jdbc_url", String.format("jdbc:snowflake://%s/", - config.get("host").asText())) - .put("host", config.get("host").asText()) + + final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:snowflake://%s/?", + config.get("host").asText())); + + // Add required properties + jdbcUrl.append(String.format("role=%s&warehouse=%s&database=%s&schema=%s&JDBC_QUERY_RESULT_FORMAT=%s&CLIENT_SESSION_KEEP_ALIVE=%s", + config.get("role").asText(), + config.get("warehouse").asText(), + config.get("database").asText(), + config.get("schema").asText(), + // Needed for JDK17 - see + // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow + "JSON", + true)); + + // https://docs.snowflake.com/en/user-guide/jdbc-configure.html#jdbc-driver-connection-string + if (config.has("jdbc_url_params")) { + jdbcUrl.append("&").append(config.get("jdbc_url_params").asText()); + } + + LOGGER.info(jdbcUrl.toString()); + + final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) .put("password", config.get("password").asText()) - .put("connection_properties", - String.format("role=%s;warehouse=%s;database=%s;schema=%s;JDBC_QUERY_RESULT_FORMAT=%s;CLIENT_SESSION_KEEP_ALIVE=%s;", - config.get("role").asText(), - config.get("warehouse").asText(), - config.get("database").asText(), - config.get("schema").asText(), - // Needed for JDK17 - see - // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow - "JSON", - true)) - .build()); + .put("jdbc_url", jdbcUrl.toString()); + + return Jsons.jsonNode(configBuilder.build()); } @Override diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json index afc06d871af63..95b989811537c 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json @@ -63,6 +63,12 @@ "airbyte_secret": true, "title": "Password", "order": 6 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 7 } } } diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java index 515981840e4a1..c9a2e59336f86 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java @@ -23,9 +23,7 @@ import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.nio.file.Path; -import java.util.Collections; import java.util.HashMap; -import java.util.List; public class SnowflakeSourceAcceptanceTest extends SourceAcceptanceTest { @@ -86,11 +84,6 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - // for each test we create a new schema in the database. run the test in there and then remove it. @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index 6cc3da7b4fbe7..2aa8e1b90f0da 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -5,21 +5,30 @@ from typing import Any, List, Mapping, Tuple from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import (AdvancedAuth, AuthFlowType, - ConnectorSpecification, - OAuthConfigSpecification, SyncMode) +from airbyte_cdk.models import AdvancedAuth, AuthFlowType, ConnectorSpecification, OAuthConfigSpecification, SyncMode from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from .spec import (CompleteOauthOutputSpecification, - CompleteOauthServerInputSpecification, - CompleteOauthServerOutputSpecification, - SourceTiktokMarketingSpec) -from .streams import (DEFAULT_START_DATE, AdGroups, AdGroupsReports, Ads, - AdsReports, Advertisers, AdvertisersReports, Campaigns, - CampaignsReports, ReportGranularity) +from .spec import ( + CompleteOauthOutputSpecification, + CompleteOauthServerInputSpecification, + CompleteOauthServerOutputSpecification, + SourceTiktokMarketingSpec, +) +from .streams import ( + DEFAULT_START_DATE, + AdGroups, + AdGroupsReports, + Ads, + AdsReports, + Advertisers, + AdvertisersReports, + Campaigns, + CampaignsReports, + ReportGranularity, +) DOCUMENTATION_URL = "https://docs.airbyte.io/integrations/sources/tiktok-marketing" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py index e393ae2ba73c6..5fa7c5d0b2b28 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py @@ -8,7 +8,6 @@ from typing import Union from jsonschema import RefResolver - from pydantic import BaseModel, Field from .streams import DEFAULT_START_DATE, ReportGranularity diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index 79434d53c0152..eded7aff2ecc4 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -9,20 +9,17 @@ from decimal import Decimal from enum import Enum from functools import total_ordering -from typing import (Any, Dict, Iterable, List, Mapping, MutableMapping, - Optional, Tuple, TypeVar, Union) +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, TypeVar, Union import pendulum -import requests - import pydantic +import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import NoAuth from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.utils.transform import (TransformConfig, - TypeTransformer) +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer # TikTok Initial release date is September 2016 DEFAULT_START_DATE = "2016-09-01" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index 594c8d3c08546..3158bfe17a4d5 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -7,12 +7,10 @@ from typing import Any, Dict, Iterable, List, Mapping, Tuple import pendulum - import pytest import requests_mock import timeout_decorator -from airbyte_cdk.sources.streams.http.exceptions import \ - UserDefinedBackoffException +from airbyte_cdk.sources.streams.http.exceptions import UserDefinedBackoffException from source_tiktok_marketing import SourceTiktokMarketing from source_tiktok_marketing.streams import Ads, Advertisers, JsonUpdatedState diff --git a/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index bfa6d4929fb2c..271744031eab5 100644 --- a/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -44,6 +44,10 @@ public static ConfiguredAirbyteCatalog createConfiguredAirbyteCatalog(final Stri return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList(createConfiguredAirbyteStream(streamName, namespace, fields))); } + public static ConfiguredAirbyteCatalog createConfiguredAirbyteCatalog(final String streamName, final String namespace, final List fields) { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList(createConfiguredAirbyteStream(streamName, namespace, fields))); + } + public static ConfiguredAirbyteStream createConfiguredAirbyteStream(final String streamName, final String namespace, final Field... fields) { return createConfiguredAirbyteStream(streamName, namespace, Arrays.asList(fields)); } diff --git a/airbyte-scheduler/app/Dockerfile b/airbyte-scheduler/app/Dockerfile index 1a5638c99153b..f5f275678db7d 100644 --- a/airbyte-scheduler/app/Dockerfile +++ b/airbyte-scheduler/app/Dockerfile @@ -5,7 +5,7 @@ ENV APPLICATION airbyte-scheduler WORKDIR /app -ADD bin/${APPLICATION}-0.35.9-alpha.tar /app +ADD bin/${APPLICATION}-0.35.13-alpha.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.13-alpha/bin/${APPLICATION}"] diff --git a/airbyte-scheduler/models/src/main/java/io/airbyte/scheduler/models/Attempt.java b/airbyte-scheduler/models/src/main/java/io/airbyte/scheduler/models/Attempt.java index d952e1dacb6a6..12f132d491c12 100644 --- a/airbyte-scheduler/models/src/main/java/io/airbyte/scheduler/models/Attempt.java +++ b/airbyte-scheduler/models/src/main/java/io/airbyte/scheduler/models/Attempt.java @@ -4,6 +4,7 @@ package io.airbyte.scheduler.models; +import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.JobOutput; import java.nio.file.Path; import java.util.Objects; @@ -16,6 +17,7 @@ public class Attempt { private final long jobId; private final JobOutput output; private final AttemptStatus status; + private final AttemptFailureSummary failureSummary; private final Path logPath; private final long updatedAtInSecond; private final long createdAtInSecond; @@ -26,6 +28,7 @@ public Attempt(final long id, final Path logPath, final @Nullable JobOutput output, final AttemptStatus status, + final @Nullable AttemptFailureSummary failureSummary, final long createdAtInSecond, final long updatedAtInSecond, final @Nullable Long endedAtInSecond) { @@ -33,6 +36,7 @@ public Attempt(final long id, this.jobId = jobId; this.output = output; this.status = status; + this.failureSummary = failureSummary; this.logPath = logPath; this.updatedAtInSecond = updatedAtInSecond; this.createdAtInSecond = createdAtInSecond; @@ -55,6 +59,10 @@ public AttemptStatus getStatus() { return status; } + public Optional getFailureSummary() { + return Optional.ofNullable(failureSummary); + } + public Path getLogPath() { return logPath; } @@ -90,13 +98,14 @@ public boolean equals(final Object o) { createdAtInSecond == attempt.createdAtInSecond && Objects.equals(output, attempt.output) && status == attempt.status && + Objects.equals(failureSummary, attempt.failureSummary) && Objects.equals(logPath, attempt.logPath) && Objects.equals(endedAtInSecond, attempt.endedAtInSecond); } @Override public int hashCode() { - return Objects.hash(id, jobId, output, status, logPath, updatedAtInSecond, createdAtInSecond, endedAtInSecond); + return Objects.hash(id, jobId, output, status, failureSummary, logPath, updatedAtInSecond, createdAtInSecond, endedAtInSecond); } @Override @@ -106,6 +115,7 @@ public String toString() { ", jobId=" + jobId + ", output=" + output + ", status=" + status + + ", failureSummary=" + failureSummary + ", logPath=" + logPath + ", updatedAtInSecond=" + updatedAtInSecond + ", createdAtInSecond=" + createdAtInSecond + diff --git a/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/AttemptTest.java b/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/AttemptTest.java index e667bda7473b2..f6c79f3a796bb 100644 --- a/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/AttemptTest.java +++ b/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/AttemptTest.java @@ -19,7 +19,7 @@ void testIsAttemptInTerminalState() { } private static Attempt attemptWithStatus(final AttemptStatus attemptStatus) { - return new Attempt(1L, 1L, null, null, attemptStatus, 0L, 0L, null); + return new Attempt(1L, 1L, null, null, attemptStatus, null, 0L, 0L, null); } } diff --git a/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/JobTest.java b/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/JobTest.java index 6f4a1a6a2c08f..5277d419f42da 100644 --- a/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/JobTest.java +++ b/airbyte-scheduler/models/src/test/java/io/airbyte/scheduler/models/JobTest.java @@ -39,7 +39,7 @@ void testHasRunningAttempt() { private static Job jobWithAttemptWithStatus(final AttemptStatus... attemptStatuses) { final List attempts = Arrays.stream(attemptStatuses) - .map(attemptStatus -> new Attempt(1L, 1L, null, null, attemptStatus, 0L, 0L, null)) + .map(attemptStatus -> new Attempt(1L, 1L, null, null, attemptStatus, null, 0L, 0L, null)) .collect(Collectors.toList()); return new Job(1L, null, null, null, attempts, null, 0L, 0L, 0L); } diff --git a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java index e748ac5f87254..048ef4279b76d 100644 --- a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java +++ b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java @@ -19,6 +19,7 @@ import io.airbyte.commons.text.Names; import io.airbyte.commons.text.Sqls; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.JobConfig; import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.config.JobOutput; @@ -99,6 +100,7 @@ public class DefaultJobPersistence implements JobPersistence { + "attempts.log_path AS log_path,\n" + "attempts.output AS attempt_output,\n" + "attempts.status AS attempt_status,\n" + + "attempts.failure_summary AS attempt_failure_summary,\n" + "attempts.created_at AS attempt_created_at,\n" + "attempts.updated_at AS attempt_updated_at,\n" + "attempts.ended_at AS attempt_ended_at\n" @@ -322,6 +324,18 @@ public void writeOutput(final long jobId, final int attemptNumber, final T o .execute()); } + @Override + public void writeAttemptFailureSummary(final long jobId, final int attemptNumber, final AttemptFailureSummary failureSummary) throws IOException { + final OffsetDateTime now = OffsetDateTime.ofInstant(timeSupplier.get(), ZoneOffset.UTC); + + jobDatabase.transaction( + ctx -> ctx.update(ATTEMPTS) + .set(ATTEMPTS.FAILURE_SUMMARY, JSONB.valueOf(Jsons.serialize(failureSummary))) + .set(ATTEMPTS.UPDATED_AT, now) + .where(ATTEMPTS.JOB_ID.eq(jobId), ATTEMPTS.ATTEMPT_NUMBER.eq(attemptNumber)) + .execute()); + } + @Override public Job getJob(final long jobId) throws IOException { return jobDatabase.query(ctx -> getJob(ctx, jobId)); @@ -441,6 +455,8 @@ private static Attempt getAttemptFromRecord(final Record record) { Path.of(record.get("log_path", String.class)), record.get("attempt_output", String.class) == null ? null : Jsons.deserialize(record.get("attempt_output", String.class), JobOutput.class), Enums.toEnum(record.get("attempt_status", String.class), AttemptStatus.class).orElseThrow(), + record.get("attempt_failure_summary", String.class) == null ? null + : Jsons.deserialize(record.get("attempt_failure_summary", String.class), AttemptFailureSummary.class), getEpoch(record, "attempt_created_at"), getEpoch(record, "attempt_updated_at"), Optional.ofNullable(record.get("attempt_ended_at")) diff --git a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java index 6abb06991082b..2cfd994cb029d 100644 --- a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java +++ b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java @@ -5,6 +5,7 @@ package io.airbyte.scheduler.persistence; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.JobConfig; import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.db.instance.jobs.JobsDatabaseSchema; @@ -125,6 +126,16 @@ public interface JobPersistence { */ void writeOutput(long jobId, int attemptNumber, T output) throws IOException; + /** + * Writes a summary of all failures that occurred during the attempt. + * + * @param jobId job id + * @param attemptNumber attempt number + * @param failureSummary summary containing failure metadata and ordered list of failures + * @throws IOException exception due to interaction with persistence + */ + void writeAttemptFailureSummary(long jobId, int attemptNumber, AttemptFailureSummary failureSummary) throws IOException; + /** * @param configTypes - type of config, e.g. sync * @param configId - id of that config diff --git a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java index 68f2d2e4fc7a1..903cdc5e7fc9a 100644 --- a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java +++ b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java @@ -21,6 +21,9 @@ import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.text.Sqls; +import io.airbyte.config.AttemptFailureSummary; +import io.airbyte.config.FailureReason; +import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.JobConfig; import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.config.JobGetSpecConfig; @@ -114,6 +117,7 @@ private static Attempt createAttempt(final long id, final long jobId, final Atte logPath, null, status, + null, NOW.getEpochSecond(), NOW.getEpochSecond(), NOW.getEpochSecond()); @@ -126,6 +130,7 @@ private static Attempt createUnfinishedAttempt(final long id, final long jobId, logPath, null, status, + null, NOW.getEpochSecond(), NOW.getEpochSecond(), null); @@ -235,6 +240,23 @@ void testWriteOutput() throws IOException { assertNotEquals(created.getAttempts().get(0).getUpdatedAtInSecond(), updated.getAttempts().get(0).getUpdatedAtInSecond()); } + @Test + @DisplayName("Should be able to read attemptFailureSummary that was written") + void testWriteAttemptFailureSummary() throws IOException { + final long jobId = jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); + final int attemptNumber = jobPersistence.createAttempt(jobId, LOG_PATH); + final Job created = jobPersistence.getJob(jobId); + final AttemptFailureSummary failureSummary = new AttemptFailureSummary().withFailures( + Collections.singletonList(new FailureReason().withFailureOrigin(FailureOrigin.SOURCE))); + + when(timeSupplier.get()).thenReturn(Instant.ofEpochMilli(4242)); + jobPersistence.writeAttemptFailureSummary(jobId, attemptNumber, failureSummary); + + final Job updated = jobPersistence.getJob(jobId); + assertEquals(Optional.of(failureSummary), updated.getAttempts().get(0).getFailureSummary()); + assertNotEquals(created.getAttempts().get(0).getUpdatedAtInSecond(), updated.getAttempts().get(0).getUpdatedAtInSecond()); + } + @Test @DisplayName("When getting the last replication job should return the most recently created job") void testGetLastSyncJobWithMultipleAttempts() throws IOException { diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 7622a7553eee3..37298e4d1b659 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -7,7 +7,7 @@ ENV APPLICATION airbyte-server WORKDIR /app -ADD bin/${APPLICATION}-0.35.9-alpha.tar /app +ADD bin/${APPLICATION}-0.35.13-alpha.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.13-alpha/bin/${APPLICATION}"] diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index ba13a25ec28c8..3266f11cc2bc2 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -32,13 +32,9 @@ import io.airbyte.scheduler.client.DefaultSchedulerJobClient; import io.airbyte.scheduler.client.DefaultSynchronousSchedulerClient; import io.airbyte.scheduler.client.SchedulerJobClient; -import io.airbyte.scheduler.models.Job; -import io.airbyte.scheduler.models.JobStatus; import io.airbyte.scheduler.persistence.DefaultJobCreator; import io.airbyte.scheduler.persistence.DefaultJobPersistence; -import io.airbyte.scheduler.persistence.JobNotifier; import io.airbyte.scheduler.persistence.JobPersistence; -import io.airbyte.scheduler.persistence.WorkspaceHelper; import io.airbyte.scheduler.persistence.job_factory.OAuthConfigSupplier; import io.airbyte.scheduler.persistence.job_tracker.JobTracker; import io.airbyte.server.errors.InvalidInputExceptionMapper; @@ -199,16 +195,6 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final Con configs.getAirbyteVersionOrWarning(), featureFlags); - if (featureFlags.usesNewScheduler()) { - final JobNotifier jobNotifier = new JobNotifier( - configs.getWebappUrl(), - configRepository, - new WorkspaceHelper(configRepository, jobPersistence), - TrackingClientSingleton.get()); - cleanupZombies(jobPersistence, jobNotifier); - migrateExistingConnection(configRepository, temporalWorkerRunFactory); - } - LOGGER.info("Starting server..."); return apiFactory.create( @@ -243,33 +229,6 @@ private static void migrateExistingConnection(final ConfigRepository configRepos LOGGER.info("Done migrating to the new scheduler..."); } - /** - * Copy paste from {@link io.airbyte.scheduler.app.SchedulerApp} which will be removed in a near - * future - * - * @param jobPersistence - * @param jobNotifier - * @throws IOException - */ - private static void cleanupZombies(final JobPersistence jobPersistence, final JobNotifier jobNotifier) throws IOException { - for (final Job zombieJob : jobPersistence.listJobsWithStatus(JobStatus.RUNNING)) { - jobNotifier.failJob("zombie job was failed", zombieJob); - - final int currentAttemptNumber = zombieJob.getAttemptsCount() - 1; - - LOGGER.warn( - "zombie clean up - job attempt was failed. job id: {}, attempt number: {}, type: {}, scope: {}", - zombieJob.getId(), - currentAttemptNumber, - zombieJob.getConfigType(), - zombieJob.getScope()); - - jobPersistence.failAttempt( - zombieJob.getId(), - currentAttemptNumber); - } - } - public static void main(final String[] args) throws Exception { try { getServer(new ServerFactory.Api(), YamlSeedConfigPersistence.getDefault()).start(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index dcf8e1fbb3475..32c349571791a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -82,6 +82,7 @@ import io.airbyte.api.model.WorkspaceRead; import io.airbyte.api.model.WorkspaceReadList; import io.airbyte.api.model.WorkspaceUpdate; +import io.airbyte.api.model.WorkspaceUpdateName; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.io.FileTtlManager; import io.airbyte.commons.version.AirbyteVersion; @@ -266,6 +267,11 @@ public WorkspaceRead updateWorkspace(final WorkspaceUpdate workspaceUpdate) { return execute(() -> workspacesHandler.updateWorkspace(workspaceUpdate)); } + @Override + public WorkspaceRead updateWorkspaceName(final WorkspaceUpdateName workspaceUpdateName) { + return execute(() -> workspacesHandler.updateWorkspaceName(workspaceUpdateName)); + } + @Override public void updateWorkspaceFeedback(final WorkspaceGiveFeedback workspaceGiveFeedback) { execute(() -> { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java index 4d04293570d63..d54f02856df08 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java @@ -20,6 +20,7 @@ import io.airbyte.api.model.WorkspaceRead; import io.airbyte.api.model.WorkspaceReadList; import io.airbyte.api.model.WorkspaceUpdate; +import io.airbyte.api.model.WorkspaceUpdateName; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -167,6 +168,21 @@ public WorkspaceRead updateWorkspace(final WorkspaceUpdate workspaceUpdate) thro return buildWorkspaceReadFromId(workspaceUpdate.getWorkspaceId()); } + public WorkspaceRead updateWorkspaceName(final WorkspaceUpdateName workspaceUpdateName) + throws JsonValidationException, ConfigNotFoundException, IOException { + final UUID workspaceId = workspaceUpdateName.getWorkspaceId(); + + final StandardWorkspace persistedWorkspace = configRepository.getStandardWorkspace(workspaceId, false); + + persistedWorkspace + .withName(workspaceUpdateName.getName()) + .withSlug(generateUniqueSlug(workspaceUpdateName.getName())); + + configRepository.writeStandardWorkspace(persistedWorkspace); + + return buildWorkspaceReadFromId(workspaceId); + } + public NotificationRead tryNotification(final Notification notification) { try { final NotificationClient notificationClient = NotificationClient.createNotificationClient(NotificationConverter.toConfig(notification)); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java index e985c0b92ac5d..ba08d917f746d 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java @@ -109,7 +109,7 @@ private static AttemptRead toAttemptRead(final Attempt a) { } private static Attempt createSuccessfulAttempt(final long jobId, final long timestamps) { - return new Attempt(ATTEMPT_ID, jobId, LOG_PATH, null, AttemptStatus.SUCCEEDED, timestamps, timestamps, timestamps); + return new Attempt(ATTEMPT_ID, jobId, LOG_PATH, null, AttemptStatus.SUCCEEDED, null, timestamps, timestamps, timestamps); } @BeforeEach diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java index aca8dd6870242..e3bb01da05249 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java @@ -28,6 +28,7 @@ import io.airbyte.api.model.WorkspaceRead; import io.airbyte.api.model.WorkspaceReadList; import io.airbyte.api.model.WorkspaceUpdate; +import io.airbyte.api.model.WorkspaceUpdateName; import io.airbyte.commons.json.Jsons; import io.airbyte.config.Notification; import io.airbyte.config.Notification.NotificationType; @@ -44,6 +45,7 @@ import java.util.UUID; import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -344,6 +346,51 @@ void testUpdateWorkspace() throws JsonValidationException, ConfigNotFoundExcepti assertEquals(expectedWorkspaceRead, actualWorkspaceRead); } + @Test + @DisplayName("Updating workspace name should update name and slug") + void testUpdateWorkspaceNoNameUpdate() throws JsonValidationException, ConfigNotFoundException, IOException { + final WorkspaceUpdateName workspaceUpdate = new WorkspaceUpdateName() + .workspaceId(workspace.getWorkspaceId()) + .name("New Workspace Name"); + + final StandardWorkspace expectedWorkspace = new StandardWorkspace() + .withWorkspaceId(workspace.getWorkspaceId()) + .withCustomerId(workspace.getCustomerId()) + .withEmail("test@airbyte.io") + .withName("New Workspace Name") + .withSlug("new-workspace-name") + .withAnonymousDataCollection(workspace.getAnonymousDataCollection()) + .withSecurityUpdates(workspace.getSecurityUpdates()) + .withNews(workspace.getNews()) + .withInitialSetupComplete(workspace.getInitialSetupComplete()) + .withDisplaySetupWizard(workspace.getDisplaySetupWizard()) + .withTombstone(false) + .withNotifications(workspace.getNotifications()); + + when(configRepository.getStandardWorkspace(workspace.getWorkspaceId(), false)) + .thenReturn(workspace) + .thenReturn(expectedWorkspace); + + final WorkspaceRead actualWorkspaceRead = workspacesHandler.updateWorkspaceName(workspaceUpdate); + + final WorkspaceRead expectedWorkspaceRead = new WorkspaceRead() + .workspaceId(workspace.getWorkspaceId()) + .customerId(workspace.getCustomerId()) + .email("test@airbyte.io") + .name("New Workspace Name") + .slug("new-workspace-name") + .initialSetupComplete(workspace.getInitialSetupComplete()) + .displaySetupWizard(workspace.getDisplaySetupWizard()) + .news(workspace.getNews()) + .anonymousDataCollection(workspace.getAnonymousDataCollection()) + .securityUpdates(workspace.getSecurityUpdates()) + .notifications(List.of(generateApiNotification())); + + verify(configRepository).writeStandardWorkspace(expectedWorkspace); + + assertEquals(expectedWorkspaceRead, actualWorkspaceRead); + } + @Test public void testSetFeedbackDone() throws JsonValidationException, ConfigNotFoundException, IOException { final WorkspaceGiveFeedback workspaceGiveFeedback = new WorkspaceGiveFeedback() diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 1377db9267439..2c7ffbd0181ea 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "airbyte-webapp", - "version": "0.35.9-alpha", + "version": "0.35.13-alpha", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "airbyte-webapp", - "version": "0.35.9-alpha", + "version": "0.35.13-alpha", "dependencies": { "@fortawesome/fontawesome-svg-core": "^1.2.36", "@fortawesome/free-brands-svg-icons": "^5.15.4", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 92e7f7dd7db7f..1d1bb4ebe502a 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.35.9-alpha", + "version": "0.35.13-alpha", "private": true, "engines": { "node": ">=16.0.0" diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 6b09a6e09e6c1..92c80d79bf39c 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -30,7 +30,7 @@ ENV APPLICATION airbyte-workers WORKDIR /app # Move worker app -ADD bin/${APPLICATION}-0.35.9-alpha.tar /app +ADD bin/${APPLICATION}-0.35.13-alpha.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.13-alpha/bin/${APPLICATION}"] diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java index 752526699bf27..fa5445a2d28df 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java @@ -4,6 +4,7 @@ package io.airbyte.workers; +import io.airbyte.config.FailureReason; import io.airbyte.config.ReplicationAttemptSummary; import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncInput; @@ -14,11 +15,13 @@ import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.config.WorkerSourceConfig; import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.workers.helper.FailureHelper; import io.airbyte.workers.protocols.airbyte.AirbyteDestination; import io.airbyte.workers.protocols.airbyte.AirbyteMapper; import io.airbyte.workers.protocols.airbyte.AirbyteSource; import io.airbyte.workers.protocols.airbyte.MessageTracker; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -27,6 +30,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,6 +108,9 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo destinationConfig.setCatalog(mapper.mapCatalog(destinationConfig.getCatalog())); final long startTime = System.currentTimeMillis(); + final AtomicReference sourceFailureRef = new AtomicReference<>(); + final AtomicReference destinationFailureRef = new AtomicReference<>(); + try { LOGGER.info("configured sync modes: {}", syncInput.getCatalog().getStreams() .stream() @@ -119,13 +126,23 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo destination.start(destinationConfig, jobRoot); source.start(sourceConfig, jobRoot); + // note: `whenComplete` is used instead of `exceptionally` so that the original exception is still + // thrown final CompletableFuture destinationOutputThreadFuture = CompletableFuture.runAsync( getDestinationOutputRunnable(destination, cancelled, messageTracker, mdc), - executors); + executors).whenComplete((msg, ex) -> { + if (ex != null) { + destinationFailureRef.set(FailureHelper.destinationFailure(ex, Long.valueOf(jobId), attempt)); + } + }); final CompletableFuture replicationThreadFuture = CompletableFuture.runAsync( getReplicationRunnable(source, destination, cancelled, mapper, messageTracker, mdc), - executors); + executors).whenComplete((msg, ex) -> { + if (ex != null) { + sourceFailureRef.set(FailureHelper.sourceFailure(ex, Long.valueOf(jobId), attempt)); + } + }); LOGGER.info("Waiting for source and destination threads to complete."); // CompletableFuture#allOf waits until all futures finish before returning, even if one throws an @@ -198,11 +215,24 @@ else if (hasFailed.get()) { .withEndTime(System.currentTimeMillis()); LOGGER.info("sync summary: {}", summary); - final ReplicationOutput output = new ReplicationOutput() .withReplicationAttemptSummary(summary) .withOutputCatalog(destinationConfig.getCatalog()); + // only .setFailures() if a failure occurred + final FailureReason sourceFailure = sourceFailureRef.get(); + final FailureReason destinationFailure = destinationFailureRef.get(); + final List failures = new ArrayList<>(); + if (sourceFailure != null) { + failures.add(sourceFailure); + } + if (destinationFailure != null) { + failures.add(destinationFailure); + } + if (!failures.isEmpty()) { + output.setFailures(failures); + } + if (messageTracker.getSourceOutputState().isPresent()) { LOGGER.info("Source output at least one state message"); } else { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index e27dd65e3130e..8aad702844c7b 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -323,6 +323,8 @@ public static void main(final String[] args) throws IOException, InterruptedExce final WorkflowServiceStubs temporalService = TemporalUtils.createTemporalService(temporalHost); + TemporalUtils.configureTemporalNamespace(temporalService); + final Database configDatabase = new ConfigsDatabaseInstance( configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/helper/FailureHelper.java b/airbyte-workers/src/main/java/io/airbyte/workers/helper/FailureHelper.java new file mode 100644 index 0000000000000..36cb450a9e1a6 --- /dev/null +++ b/airbyte-workers/src/main/java/io/airbyte/workers/helper/FailureHelper.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.helper; + +import io.airbyte.config.AttemptFailureSummary; +import io.airbyte.config.FailureReason; +import io.airbyte.config.FailureReason.FailureOrigin; +import io.airbyte.config.Metadata; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.exception.ExceptionUtils; + +public class FailureHelper { + + private static final String JOB_ID_METADATA_KEY = "jobId"; + private static final String ATTEMPT_NUMBER_METADATA_KEY = "attemptNumber"; + + private static final String WORKFLOW_TYPE_SYNC = "SyncWorkflow"; + private static final String ACTIVITY_TYPE_REPLICATE = "Replicate"; + private static final String ACTIVITY_TYPE_PERSIST = "Persist"; + private static final String ACTIVITY_TYPE_NORMALIZE = "Normalize"; + private static final String ACTIVITY_TYPE_DBT_RUN = "Run"; + + public static FailureReason genericFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return new FailureReason() + .withInternalMessage(t.getMessage()) + .withStacktrace(ExceptionUtils.getStackTrace(t)) + .withTimestamp(System.currentTimeMillis()) + .withMetadata(new Metadata() + .withAdditionalProperty(JOB_ID_METADATA_KEY, jobId) + .withAdditionalProperty(ATTEMPT_NUMBER_METADATA_KEY, attemptNumber)); + } + + public static FailureReason sourceFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.SOURCE) + .withExternalMessage("Something went wrong within the source connector"); + } + + public static FailureReason destinationFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.DESTINATION) + .withExternalMessage("Something went wrong within the destination connector"); + } + + public static FailureReason replicationWorkerFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.REPLICATION_WORKER) + .withExternalMessage("Something went wrong during replication"); + } + + public static FailureReason persistenceFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.PERSISTENCE) + .withExternalMessage("Something went wrong during state persistence"); + } + + public static FailureReason normalizationFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.NORMALIZATION) + .withExternalMessage("Something went wrong during normalization"); + } + + public static FailureReason dbtFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.DBT) + .withExternalMessage("Something went wrong during dbt"); + } + + public static FailureReason unknownOriginFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.UNKNOWN) + .withExternalMessage("An unknown failure occurred"); + } + + public static AttemptFailureSummary failureSummary(final Set failures, final Boolean partialSuccess) { + return new AttemptFailureSummary() + .withFailures(orderedFailures(failures)) + .withPartialSuccess(partialSuccess); + } + + public static FailureReason failureReasonFromWorkflowAndActivity(final String workflowType, + final String activityType, + final Throwable t, + final Long jobId, + final Integer attemptNumber) { + if (workflowType.equals(WORKFLOW_TYPE_SYNC) && activityType.equals(ACTIVITY_TYPE_REPLICATE)) { + return replicationWorkerFailure(t, jobId, attemptNumber); + } else if (workflowType.equals(WORKFLOW_TYPE_SYNC) && activityType.equals(ACTIVITY_TYPE_PERSIST)) { + return persistenceFailure(t, jobId, attemptNumber); + } else if (workflowType.equals(WORKFLOW_TYPE_SYNC) && activityType.equals(ACTIVITY_TYPE_NORMALIZE)) { + return normalizationFailure(t, jobId, attemptNumber); + } else if (workflowType.equals(WORKFLOW_TYPE_SYNC) && activityType.equals(ACTIVITY_TYPE_DBT_RUN)) { + return dbtFailure(t, jobId, attemptNumber); + } else { + return unknownOriginFailure(t, jobId, attemptNumber); + } + } + + /** + * Orders failures by timestamp, so that earlier failures come first in the list. + */ + private static List orderedFailures(final Set failures) { + return failures.stream().sorted(Comparator.comparing(FailureReason::getTimestamp)).collect(Collectors.toList()); + } + +} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index a433e4c3a8f05..db1fa4aca23fc 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.1.63"; + public static final String NORMALIZATION_VERSION = "0.1.65"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java index c2ab3170b6502..a68564f868f04 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java @@ -38,16 +38,20 @@ public TemporalCancellationHandler() { @Override public void checkAndHandleCancellation(final Runnable onCancellationCallback) { try { - /* + /** * Heartbeat is somewhat misleading here. What it does is check the current Temporal activity's * context and throw an exception if the sync has been cancelled or timed out. The input to this * heartbeat function is available as a field in thrown ActivityCompletionExceptions, which we * aren't using for now. + * + * We should use this only as a check for the ActivityCompletionException. See + * {@link TemporalUtils#withBackgroundHeartbeat} for where we actually send heartbeats to ensure + * that we don't time out the activity. */ context.heartbeat(null); } catch (final ActivityCompletionException e) { onCancellationCallback.run(); - LOGGER.warn("Job either timeout-ed or was cancelled."); + LOGGER.warn("Job either timed out or was cancelled."); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index 8d92f50dffc8c..4086d7303abbb 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -19,7 +19,6 @@ import io.temporal.activity.Activity; import java.io.IOException; import java.nio.file.Path; -import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -41,8 +40,6 @@ public class TemporalAttemptExecution implements Supplier private static final Logger LOGGER = LoggerFactory.getLogger(TemporalAttemptExecution.class); - private static final Duration HEARTBEAT_INTERVAL = Duration.ofSeconds(10); - private final JobRunConfig jobRunConfig; private final WorkerEnvironment workerEnvironment; private final LogConfigs logConfigs; @@ -135,7 +132,7 @@ public OUTPUT get() { cancellationChecker.run(); workerThread.start(); - scheduledExecutor.scheduleAtFixedRate(cancellationChecker, 0, HEARTBEAT_INTERVAL.toSeconds(), TimeUnit.SECONDS); + scheduledExecutor.scheduleAtFixedRate(cancellationChecker, 0, TemporalUtils.SEND_HEARTBEAT_INTERVAL.toSeconds(), TimeUnit.SECONDS); try { // block and wait for the output diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java index 2793f24550a2f..8e16f02539203 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java @@ -8,10 +8,15 @@ import io.airbyte.commons.lang.Exceptions; import io.airbyte.scheduler.models.JobRunConfig; +import io.temporal.activity.Activity; import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.namespace.v1.NamespaceConfig; import io.temporal.api.namespace.v1.NamespaceInfo; +import io.temporal.api.workflowservice.v1.DescribeNamespaceRequest; import io.temporal.api.workflowservice.v1.DescribeNamespaceResponse; import io.temporal.api.workflowservice.v1.ListNamespacesRequest; +import io.temporal.api.workflowservice.v1.UpdateNamespaceRequest; +import io.temporal.client.ActivityCompletionException; import io.temporal.client.WorkflowClient; import io.temporal.client.WorkflowOptions; import io.temporal.client.WorkflowStub; @@ -20,9 +25,15 @@ import io.temporal.serviceclient.WorkflowServiceStubsOptions; import io.temporal.workflow.Functions; import java.io.Serializable; +import java.time.Duration; import java.util.Set; import java.util.UUID; +import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +42,9 @@ public class TemporalUtils { private static final Logger LOGGER = LoggerFactory.getLogger(TemporalUtils.class); + public static final Duration SEND_HEARTBEAT_INTERVAL = Duration.ofSeconds(10); + public static final Duration HEARTBEAT_TIMEOUT = Duration.ofSeconds(30); + public static WorkflowServiceStubs createTemporalService(final String temporalHost) { final WorkflowServiceStubsOptions options = WorkflowServiceStubsOptions.newBuilder() // todo move to env. @@ -46,6 +60,30 @@ public static WorkflowServiceStubs createTemporalService(final String temporalHo public static final String DEFAULT_NAMESPACE = "default"; + private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(7); + private static final String HUMAN_READABLE_WORKFLOW_EXECUTION_TTL = + DurationFormatUtils.formatDurationWords(WORKFLOW_EXECUTION_TTL.toMillis(), true, true); + + public static void configureTemporalNamespace(WorkflowServiceStubs temporalService) { + final var client = temporalService.blockingStub(); + final var describeNamespaceRequest = DescribeNamespaceRequest.newBuilder().setNamespace(DEFAULT_NAMESPACE).build(); + final var currentRetentionGrpcDuration = client.describeNamespace(describeNamespaceRequest).getConfig().getWorkflowExecutionRetentionTtl(); + final var currentRetention = Duration.ofSeconds(currentRetentionGrpcDuration.getSeconds()); + + if (currentRetention.equals(WORKFLOW_EXECUTION_TTL)) { + LOGGER.info("Workflow execution TTL already set for namespace " + DEFAULT_NAMESPACE + ". Remains unchanged as: " + + HUMAN_READABLE_WORKFLOW_EXECUTION_TTL); + } else { + final var newGrpcDuration = com.google.protobuf.Duration.newBuilder().setSeconds(WORKFLOW_EXECUTION_TTL.getSeconds()).build(); + final var humanReadableCurrentRetention = DurationFormatUtils.formatDurationWords(currentRetention.toMillis(), true, true); + final var namespaceConfig = NamespaceConfig.newBuilder().setWorkflowExecutionRetentionTtl(newGrpcDuration).build(); + final var updateNamespaceRequest = UpdateNamespaceRequest.newBuilder().setNamespace(DEFAULT_NAMESPACE).setConfig(namespaceConfig).build(); + LOGGER.info("Workflow execution TTL differs for namespace " + DEFAULT_NAMESPACE + ". Changing from (" + humanReadableCurrentRetention + ") to (" + + HUMAN_READABLE_WORKFLOW_EXECUTION_TTL + "). "); + client.updateNamespace(updateNamespaceRequest); + } + } + @FunctionalInterface public interface TemporalJobCreator { @@ -144,4 +182,28 @@ protected static Set getNamespaces(final WorkflowServiceStubs temporalSe .collect(toSet()); } + /** + * Runs the code within the supplier while heartbeating in the backgroud. Also makes sure to shut + * down the heartbeat server after the fact. + */ + public static T withBackgroundHeartbeat(Callable callable) { + final ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); + + try { + scheduledExecutor.scheduleAtFixedRate(() -> { + Activity.getExecutionContext().heartbeat(null); + }, 0, SEND_HEARTBEAT_INTERVAL.toSeconds(), TimeUnit.SECONDS); + + return callable.call(); + } catch (final ActivityCompletionException e) { + LOGGER.warn("Job either timed out or was cancelled."); + throw new RuntimeException(e); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + LOGGER.info("Stopping temporal heartbeating..."); + scheduledExecutor.shutdown(); + } + } + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index ae6a25aa7dce0..e316419b08174 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -4,9 +4,11 @@ package io.airbyte.workers.temporal.scheduling; +import io.airbyte.config.FailureReason; import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; +import io.airbyte.workers.helper.FailureHelper; import io.airbyte.workers.temporal.TemporalJobType; import io.airbyte.workers.temporal.exception.RetryableException; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity; @@ -32,13 +34,16 @@ import io.airbyte.workers.temporal.scheduling.state.listener.NoopStateListener; import io.airbyte.workers.temporal.sync.SyncWorkflow; import io.temporal.api.enums.v1.ParentClosePolicy; +import io.temporal.failure.ActivityFailure; import io.temporal.failure.CanceledFailure; import io.temporal.failure.ChildWorkflowFailure; import io.temporal.workflow.CancellationScope; import io.temporal.workflow.ChildWorkflowOptions; import io.temporal.workflow.Workflow; import java.time.Duration; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; import java.util.UUID; import lombok.extern.slf4j.Slf4j; @@ -54,6 +59,8 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow Optional maybeAttemptId = Optional.empty(); Optional standardSyncOutput = Optional.empty(); + final Set failures = new HashSet<>(); + Boolean partialSuccess = null; private final GenerateInputActivity getSyncInputActivity = Workflow.newActivityStub(GenerateInputActivity.class, ActivityConfiguration.OPTIONS); private final JobCreationAndStatusUpdateActivity jobCreationAndStatusUpdateActivity = @@ -64,10 +71,13 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow private CancellationScope syncWorkflowCancellationScope; + private UUID connectionId; + public ConnectionManagerWorkflowImpl() {} @Override public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws RetryableException { + connectionId = connectionUpdaterInput.getConnectionId(); try { if (connectionUpdaterInput.getWorkflowState() != null) { workflowState = connectionUpdaterInput.getWorkflowState(); @@ -77,14 +87,18 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr // Scheduling final ScheduleRetrieverInput scheduleRetrieverInput = new ScheduleRetrieverInput( connectionUpdaterInput.getConnectionId()); + + workflowState.setResetConnection(connectionUpdaterInput.isResetConnection()); + final ScheduleRetrieverOutput scheduleRetrieverOutput = configFetchActivity.getTimeToWait(scheduleRetrieverInput); - Workflow.await(scheduleRetrieverOutput.getTimeToWait(), () -> skipScheduling() || connectionUpdaterInput.isFromFailure()); + Workflow.await(scheduleRetrieverOutput.getTimeToWait(), + () -> skipScheduling() || connectionUpdaterInput.isFromFailure()); if (!workflowState.isUpdated() && !workflowState.isDeleted()) { // Job and attempt creation maybeJobId = Optional.ofNullable(connectionUpdaterInput.getJobId()).or(() -> { final JobCreationOutput jobCreationOutput = jobCreationAndStatusUpdateActivity.createNewJob(new JobCreationInput( - connectionUpdaterInput.getConnectionId(), connectionUpdaterInput.isResetConnection())); + connectionUpdaterInput.getConnectionId(), workflowState.isResetConnection())); connectionUpdaterInput.setJobId(jobCreationOutput.getJobId()); return Optional.ofNullable(jobCreationOutput.getJobId()); }); @@ -100,7 +114,7 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr final SyncInput getSyncInputActivitySyncInput = new SyncInput( maybeAttemptId.get(), maybeJobId.get(), - connectionUpdaterInput.isResetConnection()); + workflowState.isResetConnection()); jobCreationAndStatusUpdateActivity.reportJobStart(new ReportJobStartInput( maybeJobId.get())); @@ -127,13 +141,33 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr syncWorkflowInputs.getSyncInput(), connectionId)); - StandardSyncSummary standardSyncSummary = standardSyncOutput.get().getStandardSyncSummary(); + final StandardSyncSummary standardSyncSummary = standardSyncOutput.get().getStandardSyncSummary(); + + if (workflowState.isResetConnection()) { + workflowState.setResetConnection(false); + } if (standardSyncSummary != null && standardSyncSummary.getStatus() == ReplicationStatus.FAILED) { + failures.addAll(standardSyncOutput.get().getFailures()); + partialSuccess = standardSyncSummary.getTotalStats().getRecordsCommitted() > 0; workflowState.setFailed(true); } } catch (final ChildWorkflowFailure childWorkflowFailure) { - if (!(childWorkflowFailure.getCause() instanceof CanceledFailure)) { + if (childWorkflowFailure.getCause() instanceof CanceledFailure) { + // do nothing, cancellation handled by cancellationScope + + } else if (childWorkflowFailure.getCause() instanceof ActivityFailure) { + final ActivityFailure af = (ActivityFailure) childWorkflowFailure.getCause(); + failures.add(FailureHelper.failureReasonFromWorkflowAndActivity( + childWorkflowFailure.getWorkflowType(), + af.getActivityType(), + af.getCause(), + maybeJobId.get(), + maybeAttemptId.get())); + throw childWorkflowFailure; + } else { + failures.add( + FailureHelper.unknownOriginFailure(childWorkflowFailure.getCause(), maybeJobId.get(), maybeAttemptId.get())); throw childWorkflowFailure; } } @@ -146,7 +180,9 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr // The naming is very misleading, it is not a failure but the expected behavior... } - if (connectionUpdaterInput.isResetConnection()) { + // The workflow state will be updated to true if a reset happened while a job was running. + // We need to propagate that to the new run that will be continued as new. + if (workflowState.isResetConnection()) { connectionUpdaterInput.setResetConnection(true); connectionUpdaterInput.setJobId(null); connectionUpdaterInput.setAttemptNumber(1); @@ -166,7 +202,9 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr return; } else if (workflowState.isCancelled()) { jobCreationAndStatusUpdateActivity.jobCancelled(new JobCancelledInput( - maybeJobId.get())); + maybeJobId.get(), + maybeAttemptId.get(), + failures.isEmpty() ? null : FailureHelper.failureSummary(failures, partialSuccess))); resetNewConnectionInput(connectionUpdaterInput); } else if (workflowState.isFailed()) { reportFailure(connectionUpdaterInput); @@ -196,7 +234,8 @@ private void reportFailure(final ConnectionUpdaterInput connectionUpdaterInput) jobCreationAndStatusUpdateActivity.attemptFailure(new AttemptFailureInput( connectionUpdaterInput.getJobId(), connectionUpdaterInput.getAttemptId(), - standardSyncOutput.orElse(null))); + standardSyncOutput.orElse(null), + FailureHelper.failureSummary(failures, partialSuccess))); final int maxAttempt = configFetchActivity.getMaxAttempt().getMaxAttempt(); final int attemptNumber = connectionUpdaterInput.getAttemptNumber(); @@ -208,7 +247,7 @@ private void reportFailure(final ConnectionUpdaterInput connectionUpdaterInput) } else { jobCreationAndStatusUpdateActivity.jobFailure(new JobFailureInput( connectionUpdaterInput.getJobId(), - "Job failed after too many retries")); + "Job failed after too many retries for connection " + connectionId)); Workflow.await(Duration.ofMinutes(1), () -> skipScheduling()); @@ -216,7 +255,7 @@ private void reportFailure(final ConnectionUpdaterInput connectionUpdaterInput) } } - private void resetNewConnectionInput(ConnectionUpdaterInput connectionUpdaterInput) { + private void resetNewConnectionInput(final ConnectionUpdaterInput connectionUpdaterInput) { connectionUpdaterInput.setJobId(null); connectionUpdaterInput.setAttemptNumber(1); connectionUpdaterInput.setFromFailure(false); @@ -225,7 +264,7 @@ private void resetNewConnectionInput(ConnectionUpdaterInput connectionUpdaterInp @Override public void submitManualSync() { if (workflowState.isRunning()) { - log.info("Can't schedule a manual workflow if a sync is running for this connection"); + log.info("Can't schedule a manual workflow if a sync is running for connection {}", connectionId); return; } @@ -235,7 +274,7 @@ public void submitManualSync() { @Override public void cancelJob() { if (!workflowState.isRunning()) { - log.info("Can't cancel a non-running sync"); + log.info("Can't cancel a non-running sync for connection {}", connectionId); return; } workflowState.setCancelled(true); @@ -255,10 +294,10 @@ public void connectionUpdated() { @Override public void resetConnection() { - if (!workflowState.isRunning()) { + workflowState.setResetConnection(true); + if (workflowState.isRunning()) { cancelJob(); } - workflowState.setResetConnection(true); } @Override @@ -280,7 +319,9 @@ private Boolean skipScheduling() { private void continueAsNew(final ConnectionUpdaterInput connectionUpdaterInput) { // Continue the workflow as new connectionUpdaterInput.setAttemptId(null); - boolean isDeleted = workflowState.isDeleted(); + failures.clear(); + partialSuccess = null; + final boolean isDeleted = workflowState.isDeleted(); workflowState.reset(); if (!isDeleted) { Workflow.continueAsNew(connectionUpdaterInput); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java index 33df7a447919d..aa45b53b0e8c0 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.activities; +import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.StandardSyncOutput; import io.airbyte.workers.temporal.exception.RetryableException; import io.temporal.activity.ActivityInterface; @@ -112,6 +113,7 @@ class AttemptFailureInput { private long jobId; private int attemptId; private StandardSyncOutput standardSyncOutput; + private AttemptFailureSummary attemptFailureSummary; } @@ -127,6 +129,8 @@ class AttemptFailureInput { class JobCancelledInput { private long jobId; + private int attemptId; + private AttemptFailureSummary attemptFailureSummary; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java index 04238bbf7188d..72e3dbbae542b 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java @@ -83,7 +83,7 @@ public JobCreationOutput createNewJob(final JobCreationInput input) { return new JobCreationOutput(jobId); } - } catch (JsonValidationException | ConfigNotFoundException | IOException e) { + } catch (final JsonValidationException | ConfigNotFoundException | IOException e) { throw new RetryableException(e); } } @@ -139,6 +139,7 @@ public void jobFailure(final JobFailureInput input) { public void attemptFailure(final AttemptFailureInput input) { try { jobPersistence.failAttempt(input.getJobId(), input.getAttemptId()); + jobPersistence.writeAttemptFailureSummary(input.getJobId(), input.getAttemptId(), input.getAttemptFailureSummary()); if (input.getStandardSyncOutput() != null) { final JobOutput jobOutput = new JobOutput().withSync(input.getStandardSyncOutput()); @@ -146,6 +147,7 @@ public void attemptFailure(final AttemptFailureInput input) { } else { log.warn("The job {} doesn't have any output for the attempt {}", input.getJobId(), input.getAttemptId()); } + } catch (final IOException e) { throw new RetryableException(e); } @@ -155,6 +157,9 @@ public void attemptFailure(final AttemptFailureInput input) { public void jobCancelled(final JobCancelledInput input) { try { jobPersistence.cancelJob(input.getJobId()); + if (input.getAttemptFailureSummary() != null) { + jobPersistence.writeAttemptFailureSummary(input.getJobId(), input.getAttemptId(), input.getAttemptFailureSummary()); + } final Job job = jobPersistence.getJob(input.getJobId()); trackCompletion(job, JobStatus.FAILED); jobNotifier.failJob("Job was cancelled", job); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/shared/ActivityConfiguration.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/shared/ActivityConfiguration.java index fb1919f15090e..169c327ed6d1b 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/shared/ActivityConfiguration.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/shared/ActivityConfiguration.java @@ -26,4 +26,13 @@ public class ActivityConfiguration { .setRetryOptions(TemporalUtils.NO_RETRY) .build(); + public static final ActivityOptions LONG_RUN_OPTIONS = ActivityOptions.newBuilder() + .setScheduleToCloseTimeout(Duration.ofDays(MAX_SYNC_TIMEOUT_DAYS)) + .setStartToCloseTimeout(Duration.ofDays(MAX_SYNC_TIMEOUT_DAYS)) + .setScheduleToStartTimeout(Duration.ofDays(MAX_SYNC_TIMEOUT_DAYS)) + .setCancellationType(ActivityCancellationType.WAIT_CANCELLATION_COMPLETED) + .setRetryOptions(TemporalUtils.NO_RETRY) + .setHeartbeatTimeout(TemporalUtils.HEARTBEAT_TIMEOUT) + .build(); + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index 425a915c69bb9..920541a848fb7 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -24,6 +24,7 @@ import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.temporal.CancellationHandler; import io.airbyte.workers.temporal.TemporalAttemptExecution; +import io.airbyte.workers.temporal.TemporalUtils; import java.nio.file.Path; import java.util.Optional; import java.util.function.Supplier; @@ -73,31 +74,32 @@ public Void run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig destinationLauncherConfig, final ResourceRequirements resourceRequirements, final OperatorDbtInput input) { + return TemporalUtils.withBackgroundHeartbeat(() -> { + final var fullDestinationConfig = secretsHydrator.hydrate(input.getDestinationConfiguration()); + final var fullInput = Jsons.clone(input).withDestinationConfiguration(fullDestinationConfig); - final var fullDestinationConfig = secretsHydrator.hydrate(input.getDestinationConfiguration()); - final var fullInput = Jsons.clone(input).withDestinationConfiguration(fullDestinationConfig); + final Supplier inputSupplier = () -> { + validator.ensureAsRuntime(ConfigSchema.OPERATOR_DBT_INPUT, Jsons.jsonNode(fullInput)); + return fullInput; + }; - final Supplier inputSupplier = () -> { - validator.ensureAsRuntime(ConfigSchema.OPERATOR_DBT_INPUT, Jsons.jsonNode(fullInput)); - return fullInput; - }; + final CheckedSupplier, Exception> workerFactory; - final CheckedSupplier, Exception> workerFactory; + if (containerOrchestratorConfig.isPresent()) { + workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); + } else { + workerFactory = getLegacyWorkerFactory(destinationLauncherConfig, jobRunConfig, resourceRequirements); + } - if (containerOrchestratorConfig.isPresent()) { - workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); - } else { - workerFactory = getLegacyWorkerFactory(destinationLauncherConfig, jobRunConfig, resourceRequirements); - } + final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( + workspaceRoot, workerEnvironment, logConfigs, + jobRunConfig, + workerFactory, + inputSupplier, + new CancellationHandler.TemporalCancellationHandler(), databaseUser, databasePassword, databaseUrl, airbyteVersion); - final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( - workspaceRoot, workerEnvironment, logConfigs, - jobRunConfig, - workerFactory, - inputSupplier, - new CancellationHandler.TemporalCancellationHandler(), databaseUser, databasePassword, databaseUrl, airbyteVersion); - - return temporalAttemptExecution.get(); + return temporalAttemptExecution.get(); + }); } private CheckedSupplier, Exception> getLegacyWorkerFactory(final IntegrationLauncherConfig destinationLauncherConfig, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java index 7481a5ae5b854..08b415079267f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java @@ -14,6 +14,7 @@ import io.airbyte.workers.process.AsyncOrchestratorPodProcess; import io.airbyte.workers.process.KubePodInfo; import io.airbyte.workers.process.KubeProcessFactory; +import io.airbyte.workers.temporal.TemporalUtils; import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; @@ -65,71 +66,73 @@ public LauncherWorker( @Override public OUTPUT run(INPUT input, Path jobRoot) throws WorkerException { - try { - final Map envMap = System.getenv().entrySet().stream() - .filter(entry -> OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - final Map fileMap = new HashMap<>(additionalFileMap); - fileMap.putAll(Map.of( - OrchestratorConstants.INIT_FILE_APPLICATION, application, - OrchestratorConstants.INIT_FILE_JOB_RUN_CONFIG, Jsons.serialize(jobRunConfig), - OrchestratorConstants.INIT_FILE_INPUT, Jsons.serialize(input), - OrchestratorConstants.INIT_FILE_ENV_MAP, Jsons.serialize(envMap))); - - final Map portMap = Map.of( - WorkerApp.KUBE_HEARTBEAT_PORT, WorkerApp.KUBE_HEARTBEAT_PORT, - OrchestratorConstants.PORT1, OrchestratorConstants.PORT1, - OrchestratorConstants.PORT2, OrchestratorConstants.PORT2, - OrchestratorConstants.PORT3, OrchestratorConstants.PORT3, - OrchestratorConstants.PORT4, OrchestratorConstants.PORT4); - - final var allLabels = KubeProcessFactory.getLabels( - jobRunConfig.getJobId(), - Math.toIntExact(jobRunConfig.getAttemptId()), - Collections.emptyMap()); - - final var podNameAndJobPrefix = podNamePrefix + "-j-" + jobRunConfig.getJobId() + "-a-"; - killLowerAttemptIdsIfPresent(podNameAndJobPrefix, jobRunConfig.getAttemptId()); - - final var podName = podNameAndJobPrefix + jobRunConfig.getAttemptId(); - final var kubePodInfo = new KubePodInfo(containerOrchestratorConfig.namespace(), podName); - - process = new AsyncOrchestratorPodProcess( - kubePodInfo, - containerOrchestratorConfig.documentStoreClient(), - containerOrchestratorConfig.kubernetesClient()); - - if (process.getDocStoreStatus().equals(AsyncKubePodStatus.NOT_STARTED)) { - process.create( - airbyteVersion, - allLabels, - resourceRequirements, - fileMap, - portMap); - } - - // this waitFor can resume if the activity is re-run - process.waitFor(); - - if (process.exitValue() != 0) { - throw new WorkerException("Non-zero exit code!"); - } - - final var output = process.getOutput(); - - if (output.isPresent()) { - return Jsons.deserialize(output.get(), outputClass); - } else { - throw new WorkerException("Running the " + application + " launcher resulted in no readable output!"); - } - } catch (Exception e) { - if (cancelled.get()) { - throw new WorkerException("Launcher " + application + " was cancelled.", e); - } else { - throw new WorkerException("Running the launcher " + application + " failed", e); + return TemporalUtils.withBackgroundHeartbeat(() -> { + try { + final Map envMap = System.getenv().entrySet().stream() + .filter(entry -> OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + final Map fileMap = new HashMap<>(additionalFileMap); + fileMap.putAll(Map.of( + OrchestratorConstants.INIT_FILE_APPLICATION, application, + OrchestratorConstants.INIT_FILE_JOB_RUN_CONFIG, Jsons.serialize(jobRunConfig), + OrchestratorConstants.INIT_FILE_INPUT, Jsons.serialize(input), + OrchestratorConstants.INIT_FILE_ENV_MAP, Jsons.serialize(envMap))); + + final Map portMap = Map.of( + WorkerApp.KUBE_HEARTBEAT_PORT, WorkerApp.KUBE_HEARTBEAT_PORT, + OrchestratorConstants.PORT1, OrchestratorConstants.PORT1, + OrchestratorConstants.PORT2, OrchestratorConstants.PORT2, + OrchestratorConstants.PORT3, OrchestratorConstants.PORT3, + OrchestratorConstants.PORT4, OrchestratorConstants.PORT4); + + final var allLabels = KubeProcessFactory.getLabels( + jobRunConfig.getJobId(), + Math.toIntExact(jobRunConfig.getAttemptId()), + Collections.emptyMap()); + + final var podNameAndJobPrefix = podNamePrefix + "-j-" + jobRunConfig.getJobId() + "-a-"; + killLowerAttemptIdsIfPresent(podNameAndJobPrefix, jobRunConfig.getAttemptId()); + + final var podName = podNameAndJobPrefix + jobRunConfig.getAttemptId(); + final var kubePodInfo = new KubePodInfo(containerOrchestratorConfig.namespace(), podName); + + process = new AsyncOrchestratorPodProcess( + kubePodInfo, + containerOrchestratorConfig.documentStoreClient(), + containerOrchestratorConfig.kubernetesClient()); + + if (process.getDocStoreStatus().equals(AsyncKubePodStatus.NOT_STARTED)) { + process.create( + airbyteVersion, + allLabels, + resourceRequirements, + fileMap, + portMap); + } + + // this waitFor can resume if the activity is re-run + process.waitFor(); + + if (process.exitValue() != 0) { + throw new WorkerException("Non-zero exit code!"); + } + + final var output = process.getOutput(); + + if (output.isPresent()) { + return Jsons.deserialize(output.get(), outputClass); + } else { + throw new WorkerException("Running the " + application + " launcher resulted in no readable output!"); + } + } catch (Exception e) { + if (cancelled.get()) { + throw new WorkerException("Launcher " + application + " was cancelled.", e); + } else { + throw new WorkerException("Running the launcher " + application + " failed", e); + } } - } + }); } /** diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index eb82b2c730eb8..0d6eec29979ef 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -22,6 +22,7 @@ import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.temporal.CancellationHandler; import io.airbyte.workers.temporal.TemporalAttemptExecution; +import io.airbyte.workers.temporal.TemporalUtils; import java.nio.file.Path; import java.util.Optional; import java.util.function.Supplier; @@ -70,31 +71,32 @@ public NormalizationActivityImpl(final Optional { + final var fullDestinationConfig = secretsHydrator.hydrate(input.getDestinationConfiguration()); + final var fullInput = Jsons.clone(input).withDestinationConfiguration(fullDestinationConfig); - final var fullDestinationConfig = secretsHydrator.hydrate(input.getDestinationConfiguration()); - final var fullInput = Jsons.clone(input).withDestinationConfiguration(fullDestinationConfig); + final Supplier inputSupplier = () -> { + validator.ensureAsRuntime(ConfigSchema.NORMALIZATION_INPUT, Jsons.jsonNode(fullInput)); + return fullInput; + }; - final Supplier inputSupplier = () -> { - validator.ensureAsRuntime(ConfigSchema.NORMALIZATION_INPUT, Jsons.jsonNode(fullInput)); - return fullInput; - }; + final CheckedSupplier, Exception> workerFactory; - final CheckedSupplier, Exception> workerFactory; + if (containerOrchestratorConfig.isPresent()) { + workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); + } else { + workerFactory = getLegacyWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); + } - if (containerOrchestratorConfig.isPresent()) { - workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); - } else { - workerFactory = getLegacyWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); - } + final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( + workspaceRoot, workerEnvironment, logConfigs, + jobRunConfig, + workerFactory, + inputSupplier, + new CancellationHandler.TemporalCancellationHandler(), databaseUser, databasePassword, databaseUrl, airbyteVersion); - final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( - workspaceRoot, workerEnvironment, logConfigs, - jobRunConfig, - workerFactory, - inputSupplier, - new CancellationHandler.TemporalCancellationHandler(), databaseUser, databasePassword, databaseUrl, airbyteVersion); - - return temporalAttemptExecution.get(); + return temporalAttemptExecution.get(); + }); } private CheckedSupplier, Exception> getLegacyWorkerFactory( diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java index 763f6e056e50e..304987dcfc7a1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java @@ -34,6 +34,7 @@ import io.airbyte.workers.protocols.airbyte.NamespacingMapper; import io.airbyte.workers.temporal.CancellationHandler; import io.airbyte.workers.temporal.TemporalAttemptExecution; +import io.airbyte.workers.temporal.TemporalUtils; import java.nio.file.Path; import java.util.Optional; import java.util.function.Supplier; @@ -106,47 +107,49 @@ public StandardSyncOutput replicate(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput) { - - final var fullSourceConfig = secretsHydrator.hydrate(syncInput.getSourceConfiguration()); - final var fullDestinationConfig = secretsHydrator.hydrate(syncInput.getDestinationConfiguration()); - - final var fullSyncInput = Jsons.clone(syncInput) - .withSourceConfiguration(fullSourceConfig) - .withDestinationConfiguration(fullDestinationConfig); - - final Supplier inputSupplier = () -> { - validator.ensureAsRuntime(ConfigSchema.STANDARD_SYNC_INPUT, Jsons.jsonNode(fullSyncInput)); - return fullSyncInput; - }; - - final CheckedSupplier, Exception> workerFactory; - - if (containerOrchestratorConfig.isPresent()) { - workerFactory = getContainerLauncherWorkerFactory(containerOrchestratorConfig.get(), sourceLauncherConfig, destinationLauncherConfig, - jobRunConfig, syncInput); - } else { - workerFactory = getLegacyWorkerFactory(sourceLauncherConfig, destinationLauncherConfig, jobRunConfig, syncInput); - } - - final TemporalAttemptExecution temporalAttempt = new TemporalAttemptExecution<>( - workspaceRoot, - workerEnvironment, - logConfigs, - jobRunConfig, - workerFactory, - inputSupplier, - new CancellationHandler.TemporalCancellationHandler(), - databaseUser, - databasePassword, - databaseUrl, - airbyteVersion); - - final ReplicationOutput attemptOutput = temporalAttempt.get(); - final StandardSyncOutput standardSyncOutput = reduceReplicationOutput(attemptOutput); - - LOGGER.info("sync summary: {}", standardSyncOutput); - - return standardSyncOutput; + return TemporalUtils.withBackgroundHeartbeat(() -> { + + final var fullSourceConfig = secretsHydrator.hydrate(syncInput.getSourceConfiguration()); + final var fullDestinationConfig = secretsHydrator.hydrate(syncInput.getDestinationConfiguration()); + + final var fullSyncInput = Jsons.clone(syncInput) + .withSourceConfiguration(fullSourceConfig) + .withDestinationConfiguration(fullDestinationConfig); + + final Supplier inputSupplier = () -> { + validator.ensureAsRuntime(ConfigSchema.STANDARD_SYNC_INPUT, Jsons.jsonNode(fullSyncInput)); + return fullSyncInput; + }; + + final CheckedSupplier, Exception> workerFactory; + + if (containerOrchestratorConfig.isPresent()) { + workerFactory = getContainerLauncherWorkerFactory(containerOrchestratorConfig.get(), sourceLauncherConfig, destinationLauncherConfig, + jobRunConfig, syncInput); + } else { + workerFactory = getLegacyWorkerFactory(sourceLauncherConfig, destinationLauncherConfig, jobRunConfig, syncInput); + } + + final TemporalAttemptExecution temporalAttempt = new TemporalAttemptExecution<>( + workspaceRoot, + workerEnvironment, + logConfigs, + jobRunConfig, + workerFactory, + inputSupplier, + new CancellationHandler.TemporalCancellationHandler(), + databaseUser, + databasePassword, + databaseUrl, + airbyteVersion); + + final ReplicationOutput attemptOutput = temporalAttempt.get(); + final StandardSyncOutput standardSyncOutput = reduceReplicationOutput(attemptOutput); + + LOGGER.info("sync summary: {}", standardSyncOutput); + + return standardSyncOutput; + }); } private static StandardSyncOutput reduceReplicationOutput(final ReplicationOutput output) { @@ -166,6 +169,7 @@ private static StandardSyncOutput reduceReplicationOutput(final ReplicationOutpu standardSyncOutput.setState(output.getState()); standardSyncOutput.setOutputCatalog(output.getOutputCatalog()); standardSyncOutput.setStandardSyncSummary(syncSummary); + standardSyncOutput.setFailures(output.getFailures()); return standardSyncOutput; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 4acbd8cb3319b..9a113993313ba 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -32,10 +32,11 @@ public class SyncWorkflowImpl implements SyncWorkflow { .build()) .build(); - private final ReplicationActivity replicationActivity = Workflow.newActivityStub(ReplicationActivity.class, ActivityConfiguration.OPTIONS); - private final NormalizationActivity normalizationActivity = Workflow.newActivityStub(NormalizationActivity.class, ActivityConfiguration.OPTIONS); + private final ReplicationActivity replicationActivity = Workflow.newActivityStub(ReplicationActivity.class, ActivityConfiguration.LONG_RUN_OPTIONS); + private final NormalizationActivity normalizationActivity = + Workflow.newActivityStub(NormalizationActivity.class, ActivityConfiguration.LONG_RUN_OPTIONS); private final DbtTransformationActivity dbtTransformationActivity = - Workflow.newActivityStub(DbtTransformationActivity.class, ActivityConfiguration.OPTIONS); + Workflow.newActivityStub(DbtTransformationActivity.class, ActivityConfiguration.LONG_RUN_OPTIONS); private final PersistStateActivity persistActivity = Workflow.newActivityStub(PersistStateActivity.class, persistOptions); @Override diff --git a/airbyte-workers/src/main/resources/sshtunneling.sh b/airbyte-workers/src/main/resources/sshtunneling.sh index d0961c377161f..434f8278cd2bd 100644 --- a/airbyte-workers/src/main/resources/sshtunneling.sh +++ b/airbyte-workers/src/main/resources/sshtunneling.sh @@ -31,7 +31,7 @@ function openssh() { # create a temporary file to hold ssh key and trap to delete on EXIT trap 'rm -f "$tmpkeyfile"' EXIT tmpkeyfile=$(mktemp /tmp/xyzfile.XXXXXXXXXXX) || return 1 - echo "$(cat $1 | jq -r '.tunnel_map.ssh_key')" > $tmpkeyfile + cat $1 | jq -r '.tunnel_map.ssh_key | gsub("\\\\n"; "\n")' > $tmpkeyfile # -f=background -N=no remote command -M=master mode StrictHostKeyChecking=no auto-adds host echo "Running: ssh -f -N -M -o StrictHostKeyChecking=no -S {control socket} -i {key file} -l ${tunnel_username} -L ${tunnel_local_port}:${tunnel_db_host}:${tunnel_db_port} ${tunnel_host}" ssh -f -N -M -o StrictHostKeyChecking=no -S $tmpcontrolsocket -i $tmpkeyfile -l ${tunnel_username} -L ${tunnel_local_port}:${tunnel_db_host}:${tunnel_db_port} ${tunnel_host} && diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java index f62e7b4d848e5..6d274d52b9e86 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java @@ -23,6 +23,7 @@ import io.airbyte.commons.string.Strings; import io.airbyte.config.ConfigSchema; import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.ReplicationAttemptSummary; import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSync; @@ -149,6 +150,7 @@ void testSourceNonZeroExitValue() throws Exception { final ReplicationOutput output = worker.run(syncInput, jobRoot); assertEquals(ReplicationStatus.FAILED, output.getReplicationAttemptSummary().getStatus()); + assertTrue(output.getFailures().stream().anyMatch(f -> f.getFailureOrigin().equals(FailureOrigin.SOURCE))); } @Test @@ -165,6 +167,7 @@ void testDestinationNonZeroExitValue() throws Exception { final ReplicationOutput output = worker.run(syncInput, jobRoot); assertEquals(ReplicationStatus.FAILED, output.getReplicationAttemptSummary().getStatus()); + assertTrue(output.getFailures().stream().anyMatch(f -> f.getFailureOrigin().equals(FailureOrigin.DESTINATION))); } @Test diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java index 1266b48332c9d..f7af3f7874919 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java @@ -4,11 +4,13 @@ package io.airbyte.workers.temporal.scheduling; +import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.StandardSyncInput; import io.airbyte.scheduler.models.IntegrationLauncherConfig; import io.airbyte.scheduler.models.JobRunConfig; import io.airbyte.workers.temporal.TemporalJobType; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity; +import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity.GetMaxAttemptOutput; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity.ScheduleRetrieverOutput; import io.airbyte.workers.temporal.scheduling.activities.ConnectionDeletionActivity; import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity.SyncInput; @@ -16,13 +18,19 @@ import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivityImpl; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptCreationOutput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptFailureInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCreationOutput; import io.airbyte.workers.temporal.scheduling.state.WorkflowState; import io.airbyte.workers.temporal.scheduling.state.listener.TestStateListener; import io.airbyte.workers.temporal.scheduling.state.listener.WorkflowStateChangedListener.ChangedStateEvent; import io.airbyte.workers.temporal.scheduling.state.listener.WorkflowStateChangedListener.StateField; +import io.airbyte.workers.temporal.scheduling.testsyncworkflow.DbtFailureSyncWorkflow; import io.airbyte.workers.temporal.scheduling.testsyncworkflow.EmptySyncWorkflow; +import io.airbyte.workers.temporal.scheduling.testsyncworkflow.NormalizationFailureSyncWorkflow; +import io.airbyte.workers.temporal.scheduling.testsyncworkflow.PersistFailureSyncWorkflow; +import io.airbyte.workers.temporal.scheduling.testsyncworkflow.ReplicateFailureSyncWorkflow; import io.airbyte.workers.temporal.scheduling.testsyncworkflow.SleepingSyncWorkflow; +import io.airbyte.workers.temporal.scheduling.testsyncworkflow.SourceAndDestinationFailureSyncWorkflow; import io.temporal.client.WorkflowClient; import io.temporal.client.WorkflowOptions; import io.temporal.testing.TestWorkflowEnvironment; @@ -35,17 +43,18 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatcher; import org.mockito.Mockito; public class ConnectionManagerWorkflowTest { - private static final ConfigFetchActivity mConfigFetchActivity = + private final ConfigFetchActivity mConfigFetchActivity = Mockito.mock(ConfigFetchActivity.class, Mockito.withSettings().withoutAnnotations()); - private static final ConnectionDeletionActivity mConnectionDeletionActivity = + private final ConnectionDeletionActivity mConnectionDeletionActivity = Mockito.mock(ConnectionDeletionActivity.class, Mockito.withSettings().withoutAnnotations()); - private static final GenerateInputActivityImpl mGenerateInputActivityImpl = + private final GenerateInputActivityImpl mGenerateInputActivityImpl = Mockito.mock(GenerateInputActivityImpl.class, Mockito.withSettings().withoutAnnotations()); - private static final JobCreationAndStatusUpdateActivity mJobCreationAndStatusUpdateActivity = + private final JobCreationAndStatusUpdateActivity mJobCreationAndStatusUpdateActivity = Mockito.mock(JobCreationAndStatusUpdateActivity.class, Mockito.withSettings().withoutAnnotations()); private TestWorkflowEnvironment testEnv; @@ -434,6 +443,211 @@ public void cancelRunning() { testEnv.shutdown(); } + @Test + @DisplayName("Test that resetting a-non running workflow starts a reset") + public void resetStart() { + + final UUID testId = UUID.randomUUID(); + final TestStateListener testStateListener = new TestStateListener(); + final WorkflowState workflowState = new WorkflowState(testId, testStateListener); + + final ConnectionUpdaterInput input = new ConnectionUpdaterInput( + UUID.randomUUID(), + 1L, + 1, + false, + 1, + workflowState, + false); + + WorkflowClient.start(workflow::run, input); + testEnv.sleep(Duration.ofSeconds(30L)); + workflow.resetConnection(); + testEnv.sleep(Duration.ofSeconds(90L)); + + final Queue events = testStateListener.events(testId); + + Assertions.assertThat(events) + .filteredOn(changedStateEvent -> changedStateEvent.getField() == StateField.RESET && changedStateEvent.isValue()) + .hasSizeGreaterThanOrEqualTo(1); + + Mockito.verify(mJobCreationAndStatusUpdateActivity).jobSuccess(Mockito.any()); + + testEnv.shutdown(); + } + + @Test + @DisplayName("Test that resetting a running workflow starts cancel the running workflow") + public void resetCancelRunningWorkflow() { + + final UUID testId = UUID.randomUUID(); + final TestStateListener testStateListener = new TestStateListener(); + final WorkflowState workflowState = new WorkflowState(testId, testStateListener); + + final ConnectionUpdaterInput input = new ConnectionUpdaterInput( + UUID.randomUUID(), + 1L, + 1, + false, + 1, + workflowState, + false); + + WorkflowClient.start(workflow::run, input); + workflow.submitManualSync(); + testEnv.sleep(Duration.ofSeconds(30L)); + workflow.resetConnection(); + testEnv.sleep(Duration.ofSeconds(30L)); + + final Queue events = testStateListener.events(testId); + + Assertions.assertThat(events) + .filteredOn(changedStateEvent -> changedStateEvent.getField() == StateField.RESET && changedStateEvent.isValue()) + .hasSizeGreaterThanOrEqualTo(1); + + Mockito.verify(mJobCreationAndStatusUpdateActivity).jobCancelled(Mockito.any()); + + testEnv.shutdown(); + } + + } + + @Nested + @DisplayName("Test that sync workflow failures are recorded") + class SyncWorkflowReplicationFailuresRecorded { + + private static final long JOB_ID = 111L; + private static final int ATTEMPT_ID = 222; + + @BeforeEach + public void setup() { + testEnv = TestWorkflowEnvironment.newInstance(); + worker = testEnv.newWorker(TemporalJobType.CONNECTION_UPDATER.name()); + client = testEnv.getWorkflowClient(); + worker.registerActivitiesImplementations(mConfigFetchActivity, mConnectionDeletionActivity, mGenerateInputActivityImpl, + mJobCreationAndStatusUpdateActivity); + workflow = client.newWorkflowStub(ConnectionManagerWorkflow.class, + WorkflowOptions.newBuilder().setTaskQueue(TemporalJobType.CONNECTION_UPDATER.name()).build()); + + Mockito.when(mConfigFetchActivity.getMaxAttempt()).thenReturn(new GetMaxAttemptOutput(1)); + } + + @Test + @DisplayName("Test that source and destination failures are recorded") + public void testSourceAndDestinationFailuresRecorded() { + worker.registerWorkflowImplementationTypes(ConnectionManagerWorkflowImpl.class, SourceAndDestinationFailureSyncWorkflow.class); + testEnv.start(); + + final UUID testId = UUID.randomUUID(); + final TestStateListener testStateListener = new TestStateListener(); + final WorkflowState workflowState = new WorkflowState(testId, testStateListener); + final ConnectionUpdaterInput input = new ConnectionUpdaterInput(UUID.randomUUID(), JOB_ID, ATTEMPT_ID, false, 1, workflowState, false); + + WorkflowClient.start(workflow::run, input); + testEnv.sleep(Duration.ofMinutes(2L)); + workflow.submitManualSync(); + + Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromSource(FailureOrigin.SOURCE))); + Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromSource(FailureOrigin.DESTINATION))); + + testEnv.shutdown(); + } + + @Test + @DisplayName("Test that normalization failure is recorded") + public void testNormalizationFailure() { + worker.registerWorkflowImplementationTypes(ConnectionManagerWorkflowImpl.class, NormalizationFailureSyncWorkflow.class); + testEnv.start(); + + final UUID testId = UUID.randomUUID(); + final TestStateListener testStateListener = new TestStateListener(); + final WorkflowState workflowState = new WorkflowState(testId, testStateListener); + final ConnectionUpdaterInput input = new ConnectionUpdaterInput(UUID.randomUUID(), JOB_ID, ATTEMPT_ID, false, 1, workflowState, false); + + WorkflowClient.start(workflow::run, input); + testEnv.sleep(Duration.ofMinutes(2L)); + workflow.submitManualSync(); + + Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromSource(FailureOrigin.NORMALIZATION))); + + testEnv.shutdown(); + } + + @Test + @DisplayName("Test that dbt failure is recorded") + public void testDbtFailureRecorded() { + worker.registerWorkflowImplementationTypes(ConnectionManagerWorkflowImpl.class, DbtFailureSyncWorkflow.class); + testEnv.start(); + + final UUID testId = UUID.randomUUID(); + final TestStateListener testStateListener = new TestStateListener(); + final WorkflowState workflowState = new WorkflowState(testId, testStateListener); + final ConnectionUpdaterInput input = new ConnectionUpdaterInput(UUID.randomUUID(), JOB_ID, ATTEMPT_ID, false, 1, workflowState, false); + + WorkflowClient.start(workflow::run, input); + testEnv.sleep(Duration.ofMinutes(2L)); + workflow.submitManualSync(); + + Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromSource(FailureOrigin.DBT))); + + testEnv.shutdown(); + } + + @Test + @DisplayName("Test that persistence failure is recorded") + public void testPersistenceFailureRecorded() { + worker.registerWorkflowImplementationTypes(ConnectionManagerWorkflowImpl.class, PersistFailureSyncWorkflow.class); + testEnv.start(); + + final UUID testId = UUID.randomUUID(); + final TestStateListener testStateListener = new TestStateListener(); + final WorkflowState workflowState = new WorkflowState(testId, testStateListener); + final ConnectionUpdaterInput input = new ConnectionUpdaterInput(UUID.randomUUID(), JOB_ID, ATTEMPT_ID, false, 1, workflowState, false); + + WorkflowClient.start(workflow::run, input); + testEnv.sleep(Duration.ofMinutes(2L)); + workflow.submitManualSync(); + + Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromSource(FailureOrigin.PERSISTENCE))); + + testEnv.shutdown(); + } + + @Test + @DisplayName("Test that replication worker failure is recorded") + public void testReplicationFailureRecorded() { + worker.registerWorkflowImplementationTypes(ConnectionManagerWorkflowImpl.class, ReplicateFailureSyncWorkflow.class); + testEnv.start(); + + final UUID testId = UUID.randomUUID(); + final TestStateListener testStateListener = new TestStateListener(); + final WorkflowState workflowState = new WorkflowState(testId, testStateListener); + final ConnectionUpdaterInput input = new ConnectionUpdaterInput(UUID.randomUUID(), JOB_ID, ATTEMPT_ID, false, 1, workflowState, false); + + WorkflowClient.start(workflow::run, input); + testEnv.sleep(Duration.ofMinutes(2L)); + workflow.submitManualSync(); + + Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromSource(FailureOrigin.REPLICATION_WORKER))); + + testEnv.shutdown(); + } + + } + + private class HasFailureFromSource implements ArgumentMatcher { + + private final FailureOrigin expectedFailureOrigin; + + public HasFailureFromSource(final FailureOrigin failureOrigin) { + this.expectedFailureOrigin = failureOrigin; + } + + @Override + public boolean matches(final AttemptFailureInput arg) { + return arg.getAttemptFailureSummary().getFailures().stream().anyMatch(f -> f.getFailureOrigin().equals(expectedFailureOrigin)); + } + } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java index 906124a633b08..a1fda19583880 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java @@ -4,7 +4,10 @@ package io.airbyte.workers.temporal.scheduling.activities; +import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.FailureReason; +import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.JobOutput; import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; @@ -30,6 +33,7 @@ import io.airbyte.workers.worker_run.WorkerRun; import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; import java.util.UUID; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; @@ -71,7 +75,7 @@ public class JobCreationAndStatusUpdateActivityTest { private static final UUID CONNECTION_ID = UUID.randomUUID(); private static final long JOB_ID = 123L; - private static final int ATTEMPT_ID = 321; + private static final int ATTEMPT_ID = 0; private static final StandardSyncOutput standardSyncOutput = new StandardSyncOutput() .withStandardSyncSummary( new StandardSyncSummary() @@ -79,6 +83,11 @@ public class JobCreationAndStatusUpdateActivityTest { private static final JobOutput jobOutput = new JobOutput().withSync(standardSyncOutput); + private static final AttemptFailureSummary failureSummary = new AttemptFailureSummary() + .withFailures(Collections.singletonList( + new FailureReason() + .withFailureOrigin(FailureOrigin.SOURCE))); + @Nested class Creation { @@ -186,10 +195,11 @@ public void setJobFailureWrapException() throws IOException { @Test public void setAttemptFailure() throws IOException { - jobCreationAndStatusUpdateActivity.attemptFailure(new AttemptFailureInput(JOB_ID, ATTEMPT_ID, standardSyncOutput)); + jobCreationAndStatusUpdateActivity.attemptFailure(new AttemptFailureInput(JOB_ID, ATTEMPT_ID, standardSyncOutput, failureSummary)); Mockito.verify(mJobPersistence).failAttempt(JOB_ID, ATTEMPT_ID); Mockito.verify(mJobPersistence).writeOutput(JOB_ID, ATTEMPT_ID, jobOutput); + Mockito.verify(mJobPersistence).writeAttemptFailureSummary(JOB_ID, ATTEMPT_ID, failureSummary); } @Test @@ -197,16 +207,27 @@ public void setAttemptFailureWrapException() throws IOException { Mockito.doThrow(new IOException()) .when(mJobPersistence).failAttempt(JOB_ID, ATTEMPT_ID); - Assertions.assertThatThrownBy(() -> jobCreationAndStatusUpdateActivity.attemptFailure(new AttemptFailureInput(JOB_ID, ATTEMPT_ID, null))) + Assertions + .assertThatThrownBy( + () -> jobCreationAndStatusUpdateActivity.attemptFailure(new AttemptFailureInput(JOB_ID, ATTEMPT_ID, null, failureSummary))) .isInstanceOf(RetryableException.class) .hasCauseInstanceOf(IOException.class); } @Test - public void setJobCancelled() throws IOException { - jobCreationAndStatusUpdateActivity.jobCancelled(new JobCancelledInput(JOB_ID)); + public void setJobCancelledWithNoFailures() throws IOException { + jobCreationAndStatusUpdateActivity.jobCancelled(new JobCancelledInput(JOB_ID, ATTEMPT_ID, null)); + + Mockito.verify(mJobPersistence).cancelJob(JOB_ID); + Mockito.verify(mJobPersistence, Mockito.never()).writeAttemptFailureSummary(Mockito.anyLong(), Mockito.anyInt(), Mockito.any()); + } + + @Test + public void setJobCancelledWithFailures() throws IOException { + jobCreationAndStatusUpdateActivity.jobCancelled(new JobCancelledInput(JOB_ID, ATTEMPT_ID, failureSummary)); Mockito.verify(mJobPersistence).cancelJob(JOB_ID); + Mockito.verify(mJobPersistence).writeAttemptFailureSummary(JOB_ID, ATTEMPT_ID, failureSummary); } @Test @@ -214,7 +235,7 @@ public void setJobCancelledWrapException() throws IOException { Mockito.doThrow(new IOException()) .when(mJobPersistence).cancelJob(JOB_ID); - Assertions.assertThatThrownBy(() -> jobCreationAndStatusUpdateActivity.jobCancelled(new JobCancelledInput(JOB_ID))) + Assertions.assertThatThrownBy(() -> jobCreationAndStatusUpdateActivity.jobCancelled(new JobCancelledInput(JOB_ID, ATTEMPT_ID, null))) .isInstanceOf(RetryableException.class) .hasCauseInstanceOf(IOException.class); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java new file mode 100644 index 0000000000000..982914194c599 --- /dev/null +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.temporal.scheduling.testsyncworkflow; + +import io.airbyte.config.StandardSyncInput; +import io.airbyte.config.StandardSyncOutput; +import io.airbyte.scheduler.models.IntegrationLauncherConfig; +import io.airbyte.scheduler.models.JobRunConfig; +import io.airbyte.workers.temporal.sync.SyncWorkflow; +import io.temporal.api.enums.v1.RetryState; +import io.temporal.failure.ActivityFailure; +import java.util.UUID; + +public class DbtFailureSyncWorkflow implements SyncWorkflow { + + // Should match activity types from FailureHelper.java + private static final String ACTIVITY_TYPE_DBT_RUN = "Run"; + + public static final Throwable CAUSE = new Exception("dbt failed"); + + @Override + public StandardSyncOutput run(final JobRunConfig jobRunConfig, + final IntegrationLauncherConfig sourceLauncherConfig, + final IntegrationLauncherConfig destinationLauncherConfig, + final StandardSyncInput syncInput, + final UUID connectionId) { + + throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_DBT_RUN, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); + } + +} diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java new file mode 100644 index 0000000000000..3bcc67f5cc57a --- /dev/null +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.temporal.scheduling.testsyncworkflow; + +import io.airbyte.config.StandardSyncInput; +import io.airbyte.config.StandardSyncOutput; +import io.airbyte.scheduler.models.IntegrationLauncherConfig; +import io.airbyte.scheduler.models.JobRunConfig; +import io.airbyte.workers.temporal.sync.SyncWorkflow; +import io.temporal.api.enums.v1.RetryState; +import io.temporal.failure.ActivityFailure; +import java.util.UUID; + +public class NormalizationFailureSyncWorkflow implements SyncWorkflow { + + // Should match activity types from FailureHelper.java + private static final String ACTIVITY_TYPE_NORMALIZE = "Normalize"; + + public static final Throwable CAUSE = new Exception("normalization failed"); + + @Override + public StandardSyncOutput run(final JobRunConfig jobRunConfig, + final IntegrationLauncherConfig sourceLauncherConfig, + final IntegrationLauncherConfig destinationLauncherConfig, + final StandardSyncInput syncInput, + final UUID connectionId) { + + throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_NORMALIZE, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); + } + +} diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java new file mode 100644 index 0000000000000..5dfadc3cad94b --- /dev/null +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.temporal.scheduling.testsyncworkflow; + +import io.airbyte.config.StandardSyncInput; +import io.airbyte.config.StandardSyncOutput; +import io.airbyte.scheduler.models.IntegrationLauncherConfig; +import io.airbyte.scheduler.models.JobRunConfig; +import io.airbyte.workers.temporal.sync.SyncWorkflow; +import io.temporal.api.enums.v1.RetryState; +import io.temporal.failure.ActivityFailure; +import java.util.UUID; + +public class PersistFailureSyncWorkflow implements SyncWorkflow { + + // Should match activity types from FailureHelper.java + private static final String ACTIVITY_TYPE_PERSIST = "Persist"; + + public static final Throwable CAUSE = new Exception("persist failed"); + + @Override + public StandardSyncOutput run(final JobRunConfig jobRunConfig, + final IntegrationLauncherConfig sourceLauncherConfig, + final IntegrationLauncherConfig destinationLauncherConfig, + final StandardSyncInput syncInput, + final UUID connectionId) { + + throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_PERSIST, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); + } + +} diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java new file mode 100644 index 0000000000000..9a5dc3bdbd7a4 --- /dev/null +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.temporal.scheduling.testsyncworkflow; + +import io.airbyte.config.StandardSyncInput; +import io.airbyte.config.StandardSyncOutput; +import io.airbyte.scheduler.models.IntegrationLauncherConfig; +import io.airbyte.scheduler.models.JobRunConfig; +import io.airbyte.workers.temporal.sync.SyncWorkflow; +import io.temporal.api.enums.v1.RetryState; +import io.temporal.failure.ActivityFailure; +import java.util.UUID; + +public class ReplicateFailureSyncWorkflow implements SyncWorkflow { + + // Should match activity types from FailureHelper.java + private static final String ACTIVITY_TYPE_REPLICATE = "Replicate"; + + public static final Throwable CAUSE = new Exception("replicate failed"); + + @Override + public StandardSyncOutput run(final JobRunConfig jobRunConfig, + final IntegrationLauncherConfig sourceLauncherConfig, + final IntegrationLauncherConfig destinationLauncherConfig, + final StandardSyncInput syncInput, + final UUID connectionId) { + + throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_REPLICATE, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); + } + +} diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java new file mode 100644 index 0000000000000..ceb414f2b6cef --- /dev/null +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.temporal.scheduling.testsyncworkflow; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.config.FailureReason; +import io.airbyte.config.FailureReason.FailureOrigin; +import io.airbyte.config.StandardSyncInput; +import io.airbyte.config.StandardSyncOutput; +import io.airbyte.config.StandardSyncSummary; +import io.airbyte.config.StandardSyncSummary.ReplicationStatus; +import io.airbyte.config.SyncStats; +import io.airbyte.scheduler.models.IntegrationLauncherConfig; +import io.airbyte.scheduler.models.JobRunConfig; +import io.airbyte.workers.temporal.sync.SyncWorkflow; +import java.util.Set; +import java.util.UUID; +import org.assertj.core.util.Sets; + +public class SourceAndDestinationFailureSyncWorkflow implements SyncWorkflow { + + @VisibleForTesting + public static Set FAILURE_REASONS = Sets.newLinkedHashSet( + new FailureReason().withFailureOrigin(FailureOrigin.SOURCE).withTimestamp(System.currentTimeMillis()), + new FailureReason().withFailureOrigin(FailureOrigin.DESTINATION).withTimestamp(System.currentTimeMillis())); + + @Override + public StandardSyncOutput run(final JobRunConfig jobRunConfig, + final IntegrationLauncherConfig sourceLauncherConfig, + final IntegrationLauncherConfig destinationLauncherConfig, + final StandardSyncInput syncInput, + final UUID connectionId) { + + return new StandardSyncOutput() + .withFailures(FAILURE_REASONS.stream().toList()) + .withStandardSyncSummary(new StandardSyncSummary() + .withStatus(ReplicationStatus.FAILED) + .withTotalStats(new SyncStats() + .withRecordsCommitted(10L) // should lead to partialSuccess = true + .withRecordsEmitted(20L))); + } + +} diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 183716ebfc765..8331866f83b0d 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -21,7 +21,7 @@ version: 0.3.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.35.9-alpha" +appVersion: "0.35.13-alpha" dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 907b7ed482da2..87979a5e66dcc 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -29,7 +29,7 @@ | `webapp.replicaCount` | Number of webapp replicas | `1` | | `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | | `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | -| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.35.9-alpha` | +| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.35.13-alpha` | | `webapp.podAnnotations` | Add extra annotations to the webapp pod(s) | `{}` | | `webapp.service.type` | The service type to use for the webapp service | `ClusterIP` | | `webapp.service.port` | The service port to expose the webapp on | `80` | @@ -55,7 +55,7 @@ | `scheduler.replicaCount` | Number of scheduler replicas | `1` | | `scheduler.image.repository` | The repository to use for the airbyte scheduler image. | `airbyte/scheduler` | | `scheduler.image.pullPolicy` | the pull policy to use for the airbyte scheduler image | `IfNotPresent` | -| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.35.9-alpha` | +| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.35.13-alpha` | | `scheduler.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | | `scheduler.resources.limits` | The resources limits for the scheduler container | `{}` | | `scheduler.resources.requests` | The requested resources for the scheduler container | `{}` | @@ -86,7 +86,7 @@ | `server.replicaCount` | Number of server replicas | `1` | | `server.image.repository` | The repository to use for the airbyte server image. | `airbyte/server` | | `server.image.pullPolicy` | the pull policy to use for the airbyte server image | `IfNotPresent` | -| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.35.9-alpha` | +| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.35.13-alpha` | | `server.podAnnotations` | Add extra annotations to the server pod | `{}` | | `server.livenessProbe.enabled` | Enable livenessProbe on the server | `true` | | `server.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | @@ -120,7 +120,7 @@ | `worker.replicaCount` | Number of worker replicas | `1` | | `worker.image.repository` | The repository to use for the airbyte worker image. | `airbyte/worker` | | `worker.image.pullPolicy` | the pull policy to use for the airbyte worker image | `IfNotPresent` | -| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.35.9-alpha` | +| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.35.13-alpha` | | `worker.podAnnotations` | Add extra annotations to the worker pod(s) | `{}` | | `worker.livenessProbe.enabled` | Enable livenessProbe on the worker | `true` | | `worker.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | @@ -148,7 +148,7 @@ | ----------------------------- | -------------------------------------------------------------------- | -------------------- | | `bootloader.image.repository` | The repository to use for the airbyte bootloader image. | `airbyte/bootloader` | | `bootloader.image.pullPolicy` | the pull policy to use for the airbyte bootloader image | `IfNotPresent` | -| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.35.9-alpha` | +| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.35.13-alpha` | ### Temporal parameters diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 51bd00b5760a3..a2693d387dec0 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -43,7 +43,7 @@ webapp: image: repository: airbyte/webapp pullPolicy: IfNotPresent - tag: 0.35.9-alpha + tag: 0.35.13-alpha ## @param webapp.podAnnotations [object] Add extra annotations to the webapp pod(s) ## @@ -140,7 +140,7 @@ scheduler: image: repository: airbyte/scheduler pullPolicy: IfNotPresent - tag: 0.35.9-alpha + tag: 0.35.13-alpha ## @param scheduler.podAnnotations [object] Add extra annotations to the scheduler pod ## @@ -245,7 +245,7 @@ server: image: repository: airbyte/server pullPolicy: IfNotPresent - tag: 0.35.9-alpha + tag: 0.35.13-alpha ## @param server.podAnnotations [object] Add extra annotations to the server pod ## @@ -357,7 +357,7 @@ worker: image: repository: airbyte/worker pullPolicy: IfNotPresent - tag: 0.35.9-alpha + tag: 0.35.13-alpha ## @param worker.podAnnotations [object] Add extra annotations to the worker pod(s) ## @@ -446,7 +446,7 @@ bootloader: image: repository: airbyte/bootloader pullPolicy: IfNotPresent - tag: 0.35.9-alpha + tag: 0.35.13-alpha ## @section Temporal parameters ## TODO: Move to consuming temporal from a dedicated helm chart @@ -507,7 +507,7 @@ postgresql: existingSecret: "" commonAnnotations: helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: -1 + helm.sh/hook-weight: "-1" ## External PostgreSQL configuration ## All of these values are only used when postgresql.enabled is set to false ## @param externalDatabase.host Database host diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 621ce70d95e1b..cf705f4f56372 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -14,7 +14,7 @@ * [On Azure(VM)](deploying-airbyte/on-azure-vm-cloud-shell.md) * [On GCP (Compute Engine)](deploying-airbyte/on-gcp-compute-engine.md) * [On Kubernetes (Beta)](deploying-airbyte/on-kubernetes.md) - * [On Plural (Beta)](deploying-airbyte/on-plural.md) + * [On Plural (Beta)](deploying-airbyte/on-plural.md) * [On Oracle Cloud Infrastructure VM](deploying-airbyte/on-oci-vm.md) * [On Digital Ocean Droplet](deploying-airbyte/on-digitalocean-droplet.md) * [Operator Guides](operator-guides/README.md) @@ -31,6 +31,7 @@ * [Transformations with Airbyte (Part 3/3)](operator-guides/transformation-and-normalization/transformations-with-airbyte.md) * [Configuring Airbyte](operator-guides/configuring-airbyte.md) * [Sentry Integration](operator-guides/sentry-integration.md) + * [Using Custom Connectors](operator-guides/using-custom-connectors.md) * [Scaling Airbyte](operator-guides/scaling-airbyte.md) * [Securing Airbyte](operator-guides/securing-airbyte.md) * [Connector Catalog](integrations/README.md) @@ -82,6 +83,7 @@ * [Google Workspace Admin Reports](integrations/sources/google-workspace-admin-reports.md) * [Greenhouse](integrations/sources/greenhouse.md) * [Harvest](integrations/sources/harvest.md) + * [HTTP Request (Graveyarded)](integrations/sources/http-request.md) * [HubSpot](integrations/sources/hubspot.md) * [Instagram](integrations/sources/instagram.md) * [Intercom](integrations/sources/intercom.md) @@ -263,6 +265,7 @@ * [Technical Stack](understanding-airbyte/tech-stack.md) * [Change Data Capture (CDC)](understanding-airbyte/cdc.md) * [Namespaces](understanding-airbyte/namespaces.md) + * [Supported Data Types](understanding-airbyte/supported-data-types.md) * [Json to Avro Conversion](understanding-airbyte/json-avro-conversion.md) * [Glossary of Terms](understanding-airbyte/glossary.md) * [API documentation](api-documentation.md) diff --git a/docs/deploying-airbyte/on-digitalocean-droplet.md b/docs/deploying-airbyte/on-digitalocean-droplet.md index 425a05c6f18b3..0e8159559cdb8 100644 --- a/docs/deploying-airbyte/on-digitalocean-droplet.md +++ b/docs/deploying-airbyte/on-digitalocean-droplet.md @@ -1,8 +1,7 @@ -# On Digital Ocean \(Droplet\) +# On DigitalOcean \(Droplet\) -{% hint style="info" %} -The instructions have been tested on `Digital Ocean Droplet ($5)` -{% endhint %} +The instructions have been tested on `DigitalOcean Droplet ($5)`. +##### Alternatively, you can one-click deply Airbyte to DigitalOcean using their marketplace: https://cloud.digitalocean.com/droplets/new?onboarding_origin=marketplace&appId=95451155&image=airbyte&utm_source=deploying-airbyte_on-digitalocean-droplet ## Create a new droplet @@ -32,9 +31,7 @@ The instructions have been tested on `Digital Ocean Droplet ($5)` ## Install environment -{% hint style="info" %} Note: The following commands will be entered either on your local terminal or in your ssh session on the instance terminal. The comments above each command block will indicate where to enter the commands. -{% endhint %} * Connect to your instance diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index d87b2c73143e4..07c656ce5742a 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -153,6 +153,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.6.6 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | | 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | @@ -174,6 +175,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------| :--- | +| 0.2.6 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.2.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.2.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | BigQuery/BiqQuery denorm Destinations : Support dataset-id prefixed by project-id | | 0.2.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 133e3a310c0b3..7d0001553738f 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -42,17 +42,11 @@ This mode throws an exception after receiving a configurable number of messages. ## CHANGELOG -### OSS (E2E Testing Destination) +The OSS and Cloud variants have the same version number starting from version `0.2.2`. | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--- | +| 0.2.2 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.2.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | | 0.2.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes. | | 0.1.0 | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version. | - -### Cloud (E2E Testing (`/dev/null`) Destination) - -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--- | -| 0.1.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | -| 0.1.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Create initial version. | diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 0f22a449476e0..926a811fc22be 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -111,10 +111,11 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.12 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.10 | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893)| 🎉 Destination Oracle: implemented connection encryption | +| Version | Date | Pull Request | Subject | +| :--- | :--- |:---------------------------------------------------------| :--- | +| 0.1.13 | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | +| 0.1.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| 0.1.10 | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893) | 🎉 Destination Oracle: implemented connection encryption | | 0.1.9 | 2021-10-06 | [\#6611](https://github.com/airbytehq/airbyte/pull/6611) | 🐛 Destination Oracle: maxStringLength should be 128 | | 0.1.8 | 2021-09-28 | [\#6370](https://github.com/airbytehq/airbyte/pull/6370) | Add SSH Support for Oracle Destination | | 0.1.7 | 2021-08-30 | [\#5746](https://github.com/airbytehq/airbyte/pull/5746) | Use default column name for raw tables | @@ -122,11 +123,12 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | 0.1.5 | 2021-08-10 | [\#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role | | 0.1.4 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | | 0.1.3 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.1.2 | 2021-07-20 | [4874](https://github.com/airbytehq/airbyte/pull/4874) | Require `sid` instead of `database` in connector specification | +| 0.1.2 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | Require `sid` instead of `database` in connector specification | ### Changelog (Strict Encrypt) -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:--------------------------------------------------------| :--- | +| 0.1.2 | 2021-01-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | +| 0.1.1 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | diff --git a/docs/integrations/destinations/pubsub.md b/docs/integrations/destinations/pubsub.md index 60feec7bed26a..3ad875d2dc3c6 100644 --- a/docs/integrations/destinations/pubsub.md +++ b/docs/integrations/destinations/pubsub.md @@ -89,6 +89,7 @@ Once you've configured PubSub as a destination, delete the Service Account Key f | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.2 | December 29, 2021 | [\#9183](https://github.com/airbytehq/airbyte/pull/9183) | Update connector fields title/description | | 0.1.1 | August 13, 2021 | [\#4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | | 0.1.0 | June 24, 2021 | [\#4339](https://github.com/airbytehq/airbyte/pull/4339) | Initial release | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index f1212c1abefc4..d74da8ef38369 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -113,6 +113,7 @@ You should now have all the requirements needed to configure Snowflake as a dest * **Schema** * **Username** * **Password** +* **JDBC URL Params** (Optional) ## Notes about Snowflake Naming Conventions @@ -214,9 +215,11 @@ The final query should show a `STORAGE_GCP_SERVICE_ACCOUNT` property with an ema Finally, you need to add read/write permissions to your bucket with that email. - | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | +| 0.4.7 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.4.6 | 2022-01-28 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | +| 0.4.5 | 2021-12-29 | [#9184](https://github.com/airbytehq/airbyte/pull/9184) | Update connector fields title/description | | 0.4.4 | 2022-01-24 | [#9743](https://github.com/airbytehq/airbyte/pull/9743) | Fixed bug with dashes in schema name | | 0.4.3 | 2022-01-20 | [#9531](https://github.com/airbytehq/airbyte/pull/9531) | Start using new S3StreamCopier and expose the purgeStagingData option | | 0.4.2 | 2022-01-10 | [#9141](https://github.com/airbytehq/airbyte/pull/9141) | Fixed duplicate rows on retries | @@ -236,4 +239,3 @@ Finally, you need to add read/write permissions to your bucket with that email. | 0.3.12 | 2021-07-30 | [#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | | 0.3.11 | 2021-07-21 | [#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | | 0.3.10 | 2021-07-12 | [#4713](https://github.com/airbytehq/airbyte/pull/4713)| Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake | - diff --git a/docs/integrations/sources/bigcommerce.md b/docs/integrations/sources/bigcommerce.md index 2ad2a73d04161..b62b77ddd01f8 100644 --- a/docs/integrations/sources/bigcommerce.md +++ b/docs/integrations/sources/bigcommerce.md @@ -16,6 +16,7 @@ This Source is capable of syncing the following core Streams: * [Orders](https://developer.bigcommerce.com/api-reference/store-management/orders/orders/getallorders) * [Transactions](https://developer.bigcommerce.com/api-reference/store-management/order-transactions/transactions/gettransactions) * [Pages](https://developer.bigcommerce.com/api-reference/store-management/store-content/pages/getallpages) +* [Products](https://developer.bigcommerce.com/api-reference/store-management/catalog/products/getproducts) ### Data type mapping @@ -51,6 +52,7 @@ BigCommerce has some [rate limit restrictions](https://developer.bigcommerce.com | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.4 | 2022-01-13 | [9516](https://github.com/airbytehq/airbyte/pull/9516) | Add Catalog Products Stream and fix date-time parsing | | 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | | 0.1.2 | 2021-12-07 | [8416](https://github.com/airbytehq/airbyte/pull/8416) | Correct Incremental Function | | 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | diff --git a/docs/integrations/sources/delighted.md b/docs/integrations/sources/delighted.md index 89a6db716ed5c..430ba77c977fa 100644 --- a/docs/integrations/sources/delighted.md +++ b/docs/integrations/sources/delighted.md @@ -37,6 +37,7 @@ This connector supports `API PASSWORD` as the authentication method. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-01-31 | [9550](https://github.com/airbytehq/airbyte/pull/9550) | Output only records in which cursor field is greater than the value in state for incremental streams | | 0.1.2 | 2022-01-06 | [9333](https://github.com/airbytehq/airbyte/pull/9333) | Add incremental sync mode to streams in `integration_tests/configured_catalog.json` | | 0.1.1 | 2022-01-04 | [9275](https://github.com/airbytehq/airbyte/pull/9275) | Fix pagination handling for `survey_responses`, `bounces` and `unsubscribes` streams | | 0.1.0 | 2021-10-27 | [4551](https://github.com/airbytehq/airbyte/pull/4551) | Add Delighted source connector | diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index 64a27c934f047..8d4616c359932 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -6,7 +6,7 @@ This is a mock source for testing the Airbyte pipeline. It can generate arbitrar ## Mode -### Continuous +### Continuous Feed **This is the only mode available on Airbyte Cloud.** @@ -14,6 +14,8 @@ This mode allows users to specify a single-stream or multi-stream catalog with a The single-stream catalog config exists just for convenient, since in many testing cases, one stream is enough. If only one stream is specified in the multi-stream catalog config, it is equivalent to a single-stream catalog config. +Here is its configuration: + | Mock Catalog Type | Parameters | Type | Required | Default | Notes | | --- | --- | --- | --- | --- | --- | | Single-stream | stream name | string | yes | | Name of the stream in the catalog. | @@ -23,18 +25,44 @@ The single-stream catalog config exists just for convenient, since in many testi | | random seed | integer | no | current time millis | The seed is used in random Json object generation. Min 0. Max 1 million. | | | message interval | integer | no | 0 | The time interval between messages in millisecond. Min 0 ms. Max 60000 ms (1 minute). | -## Changelog +### Legacy Infinite Feed -### OSS +This is a legacy mode used in Airbyte integration tests. It has a simple catalog with one `data` stream that has the following schema: -| Version | Date | Pull request | Notes | -| --- | --- | --- | --- | -| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. | -| 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | -| 0.1.0 | 2021-07-23 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) [\#4939](https://github.com/airbytehq/airbyte/pull/4939) | Initial release. | +```json +{ + "type": "object", + "properties": + { + "column1": { "type": "string" } + } +} +``` + +The value of `column1` will be an increasing number starting from `1`. + +This mode can generate infinite number of records, which can be dangerous. That's why it is excluded from the Cloud variant of this connector. Usually this mode should not be used. + +There are two configurable parameters: -### Cloud +| Parameters | Type | Required | Default | Notes | +| --- | --- | --- | --- | --- | +| max records | integer | no | `null` | Number of message records to emit. When it is left empty, the connector will generate infinite number of messages. | +| message interval | integer | no | `null` | Time interval between messages in millisecond. | + +### Exception after N + +This is a legacy mode used in Airbyte integration tests. It throws an `IllegalStateException` after certain number of messages. The number of messages to emit before exception is the only parameter for this mode. + +This mode is also excluded from the Cloud variant of this connector. + +## Changelog + +The OSS and Cloud variants have the same version number. The Cloud variant was initially released at version `1.0.0`. | Version | Date | Pull request | Notes | | --- | --- | --- | --- | +| 1.0.1 (unpublished) | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | +| 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | +| 0.1.0 | 2021-07-23 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) [\#4939](https://github.com/airbytehq/airbyte/pull/4939) | Initial release. | diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 2cbb098c11f41..bd64fec6c281c 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -92,6 +92,7 @@ Your token should have at least the `repo` scope. Depending on which streams you | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.15 | 2021-01-26 | [9802](https://github.com/airbytehq/airbyte/pull/9802) | Add missing fields for auto_merge in pull request stream | | 0.2.14 | 2021-01-21 | [9664](https://github.com/airbytehq/airbyte/pull/9664) | Add custom pagination size for large streams | | 0.2.13 | 2021-01-20 | [9619](https://github.com/airbytehq/airbyte/pull/9619) | Fix logging for function `should_retry` | | 0.2.11 | 2021-01-17 | [9492](https://github.com/airbytehq/airbyte/pull/9492) | Remove optional parameter `Accept` for reaction`s streams to fix error with 502 HTTP status code in response | diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index 60d14991a4094..54b527a856e8b 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -128,10 +128,21 @@ Incremental sync supports only if you add `ga:date` dimension to your custom rep The Google Analytics connector should not run into Google Analytics API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +## Sampling in reports + +Google Analytics API may, under certain circumstances, limit the returned data based on sampling. This is done to reduce the amount of data that is returned as described in https://developers.google.com/analytics/devguides/reporting/core/v4/basics#sampling +The window_in_day parameter is used to specify the number of days to look back and can be used to avoid sampling. +When sampling occurs, a warning is logged to the sync log. + +## IsDataGolden + +Google Analytics API may return provisional or incomplete data. When this occurs, the returned data will set the fleg isDataGolden to false, and the connector will log a warning to the sync log. + ## Changelog | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.16 | 2022-01-26 | [9480](https://github.com/airbytehq/airbyte/pull/9480) | Reintroduce `window_in_days` and log warning when sampling occurs | | 0.1.15 | 2021-12-28 | [9165](https://github.com/airbytehq/airbyte/pull/9165) | Update titles and descriptions | | 0.1.14 | 2021-12-09 | [8656](https://github.com/airbytehq/airbyte/pull/8656) | Fix date-format in schemas | | 0.1.13 | 2021-12-09 | [8676](https://github.com/airbytehq/airbyte/pull/8676) | Fix `window_in_days` validation issue | diff --git a/docs/integrations/sources/http-request.md b/docs/integrations/sources/http-request.md new file mode 100644 index 0000000000000..bce9807d6b8b3 --- /dev/null +++ b/docs/integrations/sources/http-request.md @@ -0,0 +1,18 @@ +# HTTP Request (Graveyarded) + +{% hint style="warning" %} +This connector is graveyarded and will not be receiving any updates from the Airbyte team. Its functionalities have been replaced by the [Airbyte CDK](../../connector-development/cdk-python/README.md), which allows you to create source connectors for any HTTP API. +{% endhint %} + +## Overview + +This connector allows you to generally connect to any HTTP API. In order to use this connector, you must manually bring it in as a custom connector. The steps to do this can be found [here](../../connector-development/tutorials/cdk-tutorial-python-http/7-use-connector-in-airbyte.md). + +## Where do I find the Docker image? + +The Docker image for the HTTP Request connector image can be found at our DockerHub [here](https://hub.docker.com/r/airbyte/source-http-request). + +## Why was this connector graveyarded? + +We found that there are lots of cases in which using a general connector leads to poor user experience, as there are countless edge cases for different API structures, different authentication policies, and varied approaches to rate-limiting. We believe that enabling users to more easily +create connectors is a more scalable and resilient approach to maximizing the quality of the user experience. \ No newline at end of file diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 1292fa2aff46b..292d7168a5b6d 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -98,9 +98,11 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | `email_events` | `content` | | `engagements` | `contacts` | | `forms` | `forms` | +| `form_submissions`| `forms` | | `line_items` | `e-commerce` | | `owners` | `contacts` | | `products` | `e-commerce` | +| `property_history` | `contacts` | | `quotes` | no scope required | | `subscription_changes` | `content` | | `tickets` | `tickets` | @@ -110,6 +112,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.36 | 2022-01-22 | [7784](https://github.com/airbytehq/airbyte/pull/7784) | Add Property History Stream | | 0.1.35 | 2021-12-24 | [9081](https://github.com/airbytehq/airbyte/pull/9081) | Add Feedback Submissions stream and update Ticket Pipelines stream | | 0.1.34 | 2022-01-20 | [9641](https://github.com/airbytehq/airbyte/pull/9641) | Add more fields for `email_events` stream | | 0.1.33 | 2022-01-14 | [8887](https://github.com/airbytehq/airbyte/pull/8887) | More efficient support for incremental updates on Companies, Contact, Deals and Engagement streams | diff --git a/docs/integrations/sources/looker.md b/docs/integrations/sources/looker.md index 1ec06e1a5f1c2..de610f5218509 100644 --- a/docs/integrations/sources/looker.md +++ b/docs/integrations/sources/looker.md @@ -78,6 +78,7 @@ Please read the "API3 Key" section in [Looker's information for users docs](http | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.7 | 2022-01-24 | [\#9609](https://github.com/airbytehq/airbyte/pull/9609) | Migrate to native CDK and fixing of intergration tests | | 0.2.6 | 2021-12-07 | [\#8578](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | | 0.2.5 | 2021-10-27 | [\#7284](https://github.com/airbytehq/airbyte/pull/7284) | Migrate Looker source to CDK structure, add SAT testing. | | 0.2.4 | 2021-06-25 | [\#3911](https://github.com/airbytehq/airbyte/pull/3911) | Added `run_look` endpoint. | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index d20309068c259..b7579593579e7 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -292,8 +292,9 @@ If you do not see a type in this list, assume that it is coerced into a string. ## Changelog -| Version | Date | Pull Request | Subject | | -|:--------| :--- | :--- | :--- | :-- | +| Version | Date | Pull Request | Subject | +|:------- | :--------- | :----------------------------------------------------- | :------------------------------------- | +| 0.3.14 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | | 0.3.13 | 2022-01-07 | [9094](https://github.com/airbytehq/airbyte/pull/9094) | Added support for missed data types | | 0.3.12 | 2021-12-30 | [9206](https://github.com/airbytehq/airbyte/pull/9206) | Update connector fields title/description | | 0.3.11 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 70b2f10d2dca6..7999c72cfd7d1 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -257,6 +257,8 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.4.4 | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description | +| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | | 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | | 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | | 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types | diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index fb3db0c36d618..8bede2da07b0e 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -23,6 +23,7 @@ The Redshift source does not alter the schema present in your warehouse. Dependi | SSL Support | Yes | | | SSH Tunnel Connection | Coming soon | | | Namespaces | Yes | Enabled by default | +| Schema Selection | Yes | Multiple schemas may be used at one time. Keep empty to process all of existing schemas | #### Incremental Sync @@ -53,6 +54,7 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.7 | 2022-01-26 | [9721](https://github.com/airbytehq/airbyte/pull/9721) | Added schema selection | | 0.3.6 | 2022-01-20 | [8617](https://github.com/airbytehq/airbyte/pull/8617) | Update connector fields title/description | | 0.3.5 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | | 0.3.4 | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only | diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index 2650bcf609255..3f663a03ed811 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -21,7 +21,9 @@ Several output streams are available from this source. A list of these streams c ### Performance considerations -The connector is restricted by normal Salesforce rate limiting. For large transfers we recommend using the BULK API. +The connector is restricted by daily Salesforce rate limiting. +The connector uses as much rate limit as it can every day, then ends the sync early with success status and continues the sync from where it left the next time. +Note that, picking up from where it ends will work only for incremental sync. ## Getting started @@ -737,6 +739,7 @@ List of available streams: | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:--------------------------------------------------------------------------| +| 0.1.21 | 2022-01-28 | [9499](https://github.com/airbytehq/airbyte/pull/9499) | If a sync reaches daily rate limit it ends the sync early with success status. Read more in `Performance considerations` section | | 0.1.20 | 2022-01-26 | [9757](https://github.com/airbytehq/airbyte/pull/9757) | Parse CSV with "unix" dialect | | 0.1.19 | 2022-01-25 | [8617](https://github.com/airbytehq/airbyte/pull/8617) | Update connector fields title/description | | 0.1.18 | 2022-01-20 | [9478](https://github.com/airbytehq/airbyte/pull/9478) | Add available stream filtering by `queryable` flag | diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index a00ef8ea605e7..4d87f62493408 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -111,6 +111,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.14 | 2022-01-26 | [9575](https://github.com/airbytehq/airbyte/pull/9575) | Correct schema | | 0.1.13 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.12 | 2021-10-07 | [6570](https://github.com/airbytehq/airbyte/pull/6570) | Implement OAuth support with OAuth authenticator | | 0.1.11 | 2021-08-27 | [5830](https://github.com/airbytehq/airbyte/pull/5830) | Fixed sync operations hang forever issue | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 17a25a3a377e8..822a526831300 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -30,7 +30,8 @@ The Snowflake source does not alter the schema present in your warehouse. Depend 6. **Schema** 7. **Username** 8. **Password** -9. Create a dedicated read-only Airbyte user and role with access to all schemas needed for replication. +9. **JDBC URL Params** (Optional) +10. Create a dedicated read-only Airbyte user and role with access to all schemas needed for replication. ### Setup guide @@ -75,9 +76,9 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2022-01-25 | [9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.1.5 | 2022-01-19 | [9567](https://github.com/airbytehq/airbyte/pull/9567) | Added parameter for keeping JDBC session alive | | 0.1.4 | 2021-12-30 | [9203](https://github.com/airbytehq/airbyte/pull/9203) | Update connector fields title/description | | 0.1.3 | 2021-01-11 | [9304](https://github.com/airbytehq/airbyte/pull/9304) | Upgrade version of JDBC driver | | 0.1.2 | 2021-10-21 | [7257](https://github.com/airbytehq/airbyte/pull/7257) | Fixed parsing of extreme values for FLOAT and NUMBER data types | | 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | - diff --git a/docs/operator-guides/configuring-airbyte-db.md b/docs/operator-guides/configuring-airbyte-db.md index 22593a228dc36..5283c41ebe102 100644 --- a/docs/operator-guides/configuring-airbyte-db.md +++ b/docs/operator-guides/configuring-airbyte-db.md @@ -89,5 +89,13 @@ The following command will allow you to access the database instance using `psql docker exec -ti airbyte-db psql -U docker -d airbyte ``` -To access the configuration files for sources, destinations, and connections that have been added, simply query the `airbyte-configs` table. +Following tables are created +1. `workspace` : Contains workspace information such as name, notification configuration, etc. +2. `actor_definition` : Contains the source and destination connector definitions. +3. `actor` : Contains source and destination connectors information. +4. `actor_oauth_parameter` : Contains source and destination oauth parameters. +5. `operation` : Contains dbt and custom normalization operations. +6. `connection` : Contains connection configuration such as catalog details, source, destination, etc. +7. `connection_operation` : Contains the operations configured for a given connection. +8. `state`. Contains the last saved state for a connection. diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index 80054797c21df..8f59dd8bdb346 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -46,7 +46,7 @@ The following variables are relevant to both Docker and Kubernetes. #### Secrets 1. `SECRET_STORE_GCP_PROJECT_ID` - Defines the GCP Project to store secrets in. Alpha support. 2. `SECRET_STORE_GCP_CREDENTIALS` - Define the JSON credentials used to read/write Airbyte Configuration to Google Secret Manager. These credentials must have Secret Manager Read/Write access. Alpha support. -3. `SECRET_PERSISTENCE_TYPE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Alpha support. Undefined behavior will result if this is turned on and then off. +3. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Alpha support. Undefined behavior will result if this is turned on and then off. #### Database 1. `DATABASE_USER` - Define the Jobs Database user. diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 67b8514de8b81..071b2ffff5119 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -101,7 +101,7 @@ If you are upgrading from \(i.e. your current version of Airbyte is\) Airbyte ve Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.35.9-alpha --\ + docker run --rm -v /tmp:/config airbyte/migration:0.35.13-alpha --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/docs/operator-guides/using-custom-connectors.md b/docs/operator-guides/using-custom-connectors.md new file mode 100644 index 0000000000000..4516f19ff9875 --- /dev/null +++ b/docs/operator-guides/using-custom-connectors.md @@ -0,0 +1,111 @@ +# Using custom connectors +If our connector catalog does not fulfill your needs, you can build your own Airbyte connectors. +There are two approaches you can take while jumping on connector development project: +1. You want to build a connector for an **external** source or destination (public API, off-the-shelf DBMS, data warehouses, etc.). In this scenario, your connector development will probably benefit the community. The right way is to open a PR on our repo to add your connector to our catalog. You will then benefit from an Airbyte team review and potential future improvements and maintenance from the community. +2. You want to build a connector for an **internal** source or destination (private API) specific to your organization. This connector has no good reason to be exposed to the community. + +This guide focuses on the second approach and assumes the following: +* You followed our other guides and tutorials about connector developments. +* You finished your connector development, running it locally on an Airbyte development instance. +* You want to deploy this connector to a production Airbyte instance running on a VM with docker-compose or on a Kubernetes cluster. + +If you prefer video tutorials, [we recorded a demo about uploading connectors images to a GCP Artifact Registry](https://www.youtube.com/watch?v=4YF20PODv30&ab_channel=Airbyte). + +## 1. Create a private Docker registry +Airbyte needs to pull its Docker images from a remote Docker registry to consume a connector. +You should host your custom connectors image on a private Docker registry. +Here are some resources to create a private Docker registry, in case your organization does not already have one: + +| Cloud provider | Service name | Documentation | +|----------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Google Cloud | Artifact Registry | [Quickstart](https://cloud.google.com/artifact-registry/docs/docker/quickstart)| +| AWS | Amazon ECR | [Getting started with Amazon ECR](https://docs.aws.amazon.com/AmazonECR/latest/userguide/getting-started-console.html)| +| Azure | Container Registry | [Quickstart](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal#:~:text=Azure%20Container%20Registry%20is%20a,container%20images%20and%20related%20artifacts.&text=Then%2C%20use%20Docker%20commands%20to,the%20image%20from%20your%20registry.)| +| DockerHub | Repositories | [DockerHub Quickstart](https://docs.docker.com/docker-hub/)| +| Self hosted | Open-source Docker Registry | [Deploy a registry server](https://docs.docker.com/registry/deploying/)| + +## 2. Authenticate to your private Docker registry +To push and pull images to your private Docker registry, you need to authenticate to it: +* Your local or CI environment (where you build your connector image) must be able to **push** images to your registry. +* Your Airbyte instance must be able to **pull** images from your registry. + +### For Docker-compose Airbyte deployments +#### On GCP - Artifact Registry: +GCP offers the `gcloud` credential helper to log in to your Artifact registry. +Please run the command detailed [here](https://cloud.google.com/artifact-registry/docs/docker/quickstart#auth) to authenticate your local environment/CI environment to your Artifact registry. +Run the same authentication flow on your Compute Engine instance. +If you do not want to use `gcloud`, GCP offers other authentication methods detailed [here](https://cloud.google.com/artifact-registry/docs/docker/authentication). + +#### On AWS - Amazon ECR: +You can authenticate to an ECR private registry using the `aws` CLI: +`aws ecr get-login-password --region region | docker login --username AWS --password-stdin aws_account_id.dkr.ecr.region.amazonaws.com` +You can find details about this command and other available authentication methods [here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/registry_auth.html). +You will have to authenticate your local/CI environment (where you build your image) **and** your EC2 instance where your Airbyte instance is running. + +#### On Azure - Container Registry: +You can authenticate to an Azure Container Registry using the `az` CLI: +`az acr login --name ` +You can find details about this command [here](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal#:~:text=Azure%20Container%20Registry%20is%20a,container%20images%20and%20related%20artifacts.&text=Then,%20use%20Docker%20commands%20to,the%20image%20from%20your%20registry.) +You will have to authenticate both your local/CI environment/ environment (where your image is built) **and** your Azure Virtual Machine instance where the Airbyte instance is running. + +#### On DockerHub - Repositories: +You can use Docker Desktop to authenticate your local machine to your DockerHub registry by signing in on the desktop application using your DockerID. +You need to use a [service account](https://docs.docker.com/docker-hub/service-accounts/) to authenticate your Airbyte instance to your DockerHub registry. + +#### Self hosted - Open source Docker Registry: +It would be best to set up auth on your Docker registry to make it private. Available authentication options for an open-source Docker registry are listed [here](https://docs.docker.com/registry/configuration/#auth). +To authenticate your local/CI environment and Airbyte instance you can use the [`docker login`](https://docs.docker.com/engine/reference/commandline/login/) command. + +### For Kubernetes Airbyte deployments +You can use the previous section's authentication flow to authenticate your local/CI to your private Docker registry. +If you provisioned your Kubernetes cluster using AWS EKS, GCP GKE, or Azure AKS: it is very likely that you already allowed your cluster to pull images from the respective container registry service of your cloud provider. +If you want Airbyte to pull images from another private Docker registry, you will have to do the following: +1. Create a `Secret` in Kubernetes that will host your authentication credentials. [This Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) explains how to proceed. +2. Set the `JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET` environment variable on the `airbyte-worker` pod. The value must be **the name of your previously created Kubernetes Secret**. + +## 3. Push your connector image to your private Docker registry +1. Build and tag your connector image locally, e.g.: `docker build . -t my-custom-connectors/source-custom:0.1.0` +2. Create your image tag with `docker tag` command. The structure of the remote tag depends on your cloud provider's container registry service. Please check their online documentation linked at the top. +3. Use `docker push :` to push the image to your private Docker registry. + +You should run all the above commands from your local/CI environment, where your connector source code is available. + +## 4. Use your custom connector in Airbyte +At this step, you should have: +* A private Docker registry hosting your custom connector image. +* Authenticated your Airbyte instance to your private Docker registry. + +You can pull your connector image from your private registry to validate the previous steps. On your Airbyte instance: run `docker pull :` if you are using our `docker-compose` deployment, or start a pod that is using the connector image. + +### 1. Click on Settings +![Step 1 screenshot](https://images.tango.us/public/screenshot_bf5c3e27-19a3-4cc0-bc40-90c80afdbcba?crop=focalpoint&fit=crop&fp-x=0.0211&fp-y=0.9320&fp-z=2.9521&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) + + +### 2. Click on Sources (or Destinations) +![Step 2 screenshot](https://images.tango.us/public/screenshot_d956e987-424d-4f76-ad39-f6d6172f6acc?crop=focalpoint&fit=crop&fp-x=0.0855&fp-y=0.1083&fp-z=2.7473&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) + + +### 3. Click on + New connector +![Step 3 screenshot](https://images.tango.us/public/screenshot_52248202-6351-496d-bc8f-892c43cf7cf8?crop=focalpoint&fit=crop&fp-x=0.8912&fp-y=0.0833&fp-z=3.0763&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) + + +### 4. Fill the name of your custom connector +![Step 4 screenshot](https://images.tango.us/public/screenshot_809a22c8-ff38-4b10-8292-bce7364f111c?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.4145&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) + + +### 5. Fill the Docker image name of your custom connector +![Step 5 screenshot](https://images.tango.us/public/screenshot_ed91d789-9fc7-4758-a6f0-50bf2f04f248?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.4924&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) + + +### 6. Fill the Docker Tag of your custom connector image +![Step 6 screenshot](https://images.tango.us/public/screenshot_5b6bff70-5703-4dac-b359-95b9ab8f8ce1?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.5703&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) + + +### 7. Fill the URL to your connector documentation +This is a required field at the moment, but you can fill with any value if you do not have online documentation for your connector. +This documentation will be linked in the connector setting page. +![Step 7 screenshot](https://images.tango.us/public/screenshot_007e6465-619f-4553-8d65-9af2f5ad76bc?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.6482&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) + + +### 8. Click on Add +![Step 8 screenshot](https://images.tango.us/public/screenshot_c097183f-1687-469f-852d-f66f743e8c10?crop=focalpoint&fit=crop&fp-x=0.5968&fp-y=0.7010&fp-z=3.0725&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 457677dd409a7..e2037ab882016 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -200,6 +200,7 @@

Airbyte Configuration API

  • Adding new HTTP endpoints.
    • +
  • All web_backend APIs are not considered public APIs and are not guaranteeing backwards compatibility.
    • More information: https://openapi-generator.tech
    @@ -358,6 +359,7 @@

    Workspace

  • post /v1/workspaces/list
  • post /v1/workspaces/update
  • post /v1/workspaces/tag_feedback_status_as_done
    • +
  • post /v1/workspaces/update_name

    • Connection

    @@ -7446,6 +7448,88 @@

    404

    NotFoundKnownExceptionInfo
    +
    +
    + Up + 
    post /v1/workspaces/update_name
    +
    Update workspace name (updateWorkspaceName)
    +
    + + +

    Consumes

    + This API call consumes the following media types via the Content-Type request header: +
      +
    • application/json
      • +
    + +

    Request body

    +
    +
    WorkspaceUpdateName WorkspaceUpdateName (required)
    + +
    Body Parameter —
    + +
    + + + + +

    Return type

    +
    + WorkspaceRead + +
    + + + +

    Example data

    +
    Content-Type: application/json
    + 
    {
+  "news" : true,
+  "displaySetupWizard" : true,
+  "initialSetupComplete" : true,
+  "anonymousDataCollection" : true,
+  "customerId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "name" : "name",
+  "firstCompletedSync" : true,
+  "feedbackDone" : true,
+  "email" : "email",
+  "slug" : "slug",
+  "securityUpdates" : true,
+  "notifications" : [ {
+    "slackConfiguration" : {
+      "webhook" : "webhook"
+    },
+    "sendOnSuccess" : false,
+    "sendOnFailure" : true
+  }, {
+    "slackConfiguration" : {
+      "webhook" : "webhook"
+    },
+    "sendOnSuccess" : false,
+    "sendOnFailure" : true
+  } ],
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    + +

    Produces

    + This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
      +
    • application/json
      • +
    + +

    Responses

    +

    200

    + Successful operation + WorkspaceRead +

    404

    + Object with given id was not found. + NotFoundKnownExceptionInfo +

    422

    + Input failed validation + InvalidInputExceptionInfo +
    +

    Models

    [ Jump to Methods ] @@ -7570,6 +7654,7 @@

    Table of Contents

  • WorkspaceRead -
  • WorkspaceReadList -
  • WorkspaceUpdate -
    • +
  • WorkspaceUpdateName -
    • @@ -8710,5 +8795,13 @@

    WorkspaceUpdate - notifications (optional)

    array[Notification]
    +
    +

    WorkspaceUpdateName - Up

    +
    +
    +
    workspaceId
    UUID format: uuid
    +
    name
    String
    +
    +
    diff --git a/docs/troubleshooting/README.md b/docs/troubleshooting/README.md index 43c736ddbd468..2599cbc739caf 100644 --- a/docs/troubleshooting/README.md +++ b/docs/troubleshooting/README.md @@ -9,7 +9,7 @@ The troubleshooting section is aimed at collecting common issues users have to p * [On Running a Sync](running-sync.md) * [On Upgrading](on-upgrading.md) -If you don't see your issue listed in those sections, you can send a message in our \#issues Slack channel. Using the template bellow will allow us to address your issue quickly and will give us full understanding of your situation. +If you don't see your issue listed in those sections, you can send a message in our \#troubleshooting Slack channel. Using the template bellow will allow us to address your issue quickly and will give us full understanding of your situation. ## Slack Issue Template diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 7d8dac248665d..bc2dc591bbd0b 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -350,6 +350,8 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | :--- | +| 0.35.13-alpha | 0.1.65 | 2021-01-28 | [\#9846](https://github.com/airbytehq/airbyte/pull/9846) | Tweak dbt multi-thread parameter down | +| 0.35.12-alpha | 0.1.64 | 2021-01-28 | [\#9793](https://github.com/airbytehq/airbyte/pull/9793) | Support PEM format for ssh-tunnel keys | | 0.35.4-alpha | 0.1.63 | 2021-01-07 | [\#9301](https://github.com/airbytehq/airbyte/pull/9301) | Fix Snowflake prefix tables starting with numbers | | | 0.1.62 | 2021-01-07 | [\#9340](https://github.com/airbytehq/airbyte/pull/9340) | Use TCP-port support for clickhouse | | | 0.1.62 | 2021-01-07 | [\#9063](https://github.com/airbytehq/airbyte/pull/9063) | Change Snowflake-specific materialization settings | diff --git a/docs/understanding-airbyte/supported-data-types.md b/docs/understanding-airbyte/supported-data-types.md new file mode 100644 index 0000000000000..cdb56a07a3832 --- /dev/null +++ b/docs/understanding-airbyte/supported-data-types.md @@ -0,0 +1,166 @@ +# Data Types in Records + +AirbyteRecords are required to conform to the Airbyte type system. This means that all sources must produce schemas and records within these types, and all destinations must handle records that conform to this type system. + +Because Airybet's interfaces are JSON-based, this type system is realized using [JSON schemas](https://json-schema.org/). In order to work around some linmitations of JSON schemas, schemas may declare an additional `airbyte_type` annotation. This is used to disambiguate certain types that JSON schema does not explicitly differentiate between. See the [specific types](#specific-types) section for details. + +This type system does not (generally) constrain values. Sources may declare streams using additional features of JSON schema (such as the `length` property for strings), but those constraints will be ignored by all other Airbyte components. The exception is in numeric types; `integer` and `number` fields must be representable within 64-bit primitives. + +## The types + +This table summarizes the available types. See the [Specific Types](#specific-types) section for explanation of optional parameters. + +| Airbyte type | JSON Schema | Examples | +| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| String | `{"type": "string"}` | `"foo bar"` | +| Date | `{"type": "string", "format": "date"}` | `"2021-01-23"` | +| Datetime with timezone | `{"type": "string", "format": "datetime", "airbyte_type": "timestamp_with_timezone"}` | `"2022-11-22T01:23:45+05:00"` | +| Datetime without timezone | `{"type": "string", "format": "datetime", "airbyte_type": "timestamp_without_timezone"}` | `"2022-11-22T01:23:45"` | +| Integer | `{"type": "integer"}` | `42` | +| Big integer (unrepresentable as a 64-bit two's complement int) | `{"type": "string", "airbyte_type": "big_integer"}` | `"123141241234124123141241234124123141241234124123141241234124123141241234124"` | +| Number | `{"type": "number"}` | `1234.56` | +| Big number (unrepresentable as a 64-bit IEEE 754 float) | `{"type": "string", "airbyte_type": "big_number"}` | `"1,000,000,...,000.1234"` with 500 0's | +| Array | `{"type": "array"}`; optionally `items` and `additionalItems` | `[1, 2, 3]` | +| Object | `{"type": "object"}`; optionally `properties` and `additionalProperties` | `{"foo": "bar"}` | +| Untyped (i.e. any value is valid) | `{}` | | +| Union | `{"anyOf": [...]}` or `{"oneOf": [...]}` | | + +### Record structure +As a reminder, sources expose a `discover` command, which returns a list of [`AirbyteStreams`](https://github.com/airbytehq/airbyte/blob/111131a193359027d0081de1290eb4bb846662ef/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L122), and a `read` method, which emits a series of [`AirbyteRecordMessages`](https://github.com/airbytehq/airbyte/blob/111131a193359027d0081de1290eb4bb846662ef/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L46-L66). The type system determines what a valid `json_schema` is for an `AirbyteStream`, which in turn dictates what messages `read` is allowed to emit. + +For example, a source could produce this `AirbyteStream` (remember that the `json_schema` must declare `"type": "object"` at the top level): +```json +{ + "name": "users", + "json_schema": { + "type": "object", + "properties": { + "username": { + "type": "string" + }, + "age": { + "type": "integer" + }, + "appointments": { + "type": "array", + "items": "string", + "airbyte_type": "timestamp_with_timezone" + } + } + } +} +``` +Along with this `AirbyteRecordMessage` (observe that the `data` field conforms to the `json_schema` from the stream): +```json +{ + "stream": "users", + "data": { + "username": "someone42", + "age": 84, + "appointments": ["2021-11-22T01:23:45+00:00", "2022-01-22T14:00:00+00:00"] + }, + "emitted_at": 1623861660 +} +``` + +The top-level `object` must conform to the type system. This [means](#objects) that all of the fields must also conform to the type system. + +#### Nulls +Many sources cannot guarantee that all fields are present on all records. As such, they may replace the `type` entry in the schema with `["null", "the_real_type"]`. For example, this schema is the correct way for a source to declare that the `age` field may be missing from some records: +```json +{ + "type": "object", + "properties": { + "username": { + "type": "string" + }, + "age": { + "type": ["null", "integer"] + } + } +} +``` +This would then be a valid record: +```json +{"username": "someone42"} +``` + +Nullable fields are actually the more common case, but this document omits them in other examples for the sake of clarity. + +#### Unsupported types +As an escape hatch, destinations which cannot handle a certain type should just fall back to treating those values as strings. For example, let's say a source discovers a stream with this schema: +```json +{ + "type": "object", + "properties": { + "appointments": { + "type": "array", + "items": { + "type": "string", + "airbyte_type": "timestamp_with_timezone" + } + } + } +} +``` +Along with records that look like this: +```json +{"appointments": ["2021-11-22T01:23:45+00:00", "2022-01-22T14:00:00+00:00"]} +``` + +The user then connects this source to a destination that cannot handle `array` fields. The destination connector should simply JSON-serialize the array back to a string when pushing data into the end platform. In other words, the destination connector should behave as though the source declared this schema: +```json +{ + "type": "object", + "properties": { + "appointments": { + "type": "string" + } + } +} +``` +And emitted this record: +```json +{"appointments": "[\"2021-11-22T01:23:45+00:00\", \"2022-01-22T14:00:00+00:00\"]"} +``` + +### Specific types + +#### Dates and timestamps +Airbyte has three temporal types: `date`, `timestamp_with_timezone`, and `timestamp_without_timezone`. These are represented as strings with specific `format` (either `date` or `date-time`). + +However, JSON schema does not have a built-in way to indicate whether a field includes timezone information. For example, given the schema +```json +{ + "type": "object", + "properties": { + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} +``` +Both `{"created_at": "2021-11-22T01:23:45+00:00"}` and `{"created_at": "2021-11-22T01:23:45"}` are valid records. The `airbyte_type` annotation resolves this ambiguity; sources producing `date-time` fields **must** set the `airbyte_type` to either `timestamp_with_timezone` or `timestamp_without_timezone`. + +#### Unrepresentable numbers +64-bit integers and floating-point numbers (AKA `long` and `double`) cannot represent every number in existence. The `big_integer` and `big_number` types indicate that the source may produce numbers outside the ranges of `long` and `double`s. + +Note that these are declared as `"type": "string"`. This is intended to make parsing more safe by preventing accidental overflow/loss-of-precision. + +#### Arrays +Arrays contain 0 or more items, which must have a defined type. These types should also conform to the type system. Arrays may require that all of their elements be the same type (`"items": {whatever type...}`), or they may require specific types for the first N entries (`"items": [{first type...}, {second type...}, ... , {Nth type...}]`, AKA tuple-type). + +Tuple-typed arrays can configure the type of any additional elements using the `additionalItems` field; by default, any type is allowed. They may also pass a boolean to enable/disable additional elements, with `"additionalItems": true` being equivalent to `"additionalItems": {}` and `"additionalItems": false` meaning that only the tuple-defined items are allowed. + +Destinations may have a difficult time supporting tuple-typed arrays without very specific handling, and as such are permitted to somewhat loosen their requirements. For example, many Avro-based destinations simply declare an array of a union of all allowed types, rather than requiring the correct type in each position of the array. + +#### Objects +As with arrays, objects may declare `properties`, each of which should have a type which conforms to the type system. Objects may additionally accept `additionalProperties`, as `true` (any type is acceptable), a specific type (all additional properties must be of that type), or `false` (no additonal properties are allowed). + +#### Unions +In some cases, sources may want to use multiple types for the same field. For example, a user might have a property which holds either an object, or a `string` explanation of why that data is missing. This is supported with JSON schema's `oneOf` and `anyOf` types. + +#### Untyped values +In some unusual cases, a property may not have type information associated with it. This is represented by the empty schema `{}`. As many destinations do not allow untyped data, this will frequently trigger the [string-typed escape hatch](#unsupported-types). diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 07a059ad694a6..b10ab27b77866 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.35.9-alpha +AIRBYTE_VERSION=0.35.13-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml index 1c0e8ededf1e0..7f7c7ffdbd626 100644 --- a/kube/overlays/stable-with-resource-limits/kustomization.yaml +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -8,17 +8,17 @@ bases: images: - name: airbyte/db - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/bootloader - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/scheduler - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/server - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/webapp - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/worker - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 07a059ad694a6..b10ab27b77866 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.35.9-alpha +AIRBYTE_VERSION=0.35.13-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index 1d18f0b2b5159..e58abd8e4f780 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -8,17 +8,17 @@ bases: images: - name: airbyte/db - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/bootloader - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/scheduler - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/server - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/webapp - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: airbyte/worker - newTag: 0.35.9-alpha + newTag: 0.35.13-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/tools/code-generator/Dockerfile b/tools/code-generator/Dockerfile index 94b2d74630ca9..47ab14abb31e1 100644 --- a/tools/code-generator/Dockerfile +++ b/tools/code-generator/Dockerfile @@ -1,5 +1,7 @@ FROM python:3.10-slim -RUN pip install datamodel-code-generator==0.10.1 +# pin black to version 21.12b0 because its latest version +# 22.1.0 seems incompatible with datamodel-code-generator +RUN pip install black==21.12b0 datamodel-code-generator==0.10.1 ENTRYPOINT ["datamodel-codegen"] LABEL io.airbyte.version=dev diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index f6ae667e3a240..3cba9dd7d2c7b 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -13,6 +13,7 @@ Available commands: scaffold build [] publish [] [--publish_spec_to_cache] [--publish_spec_to_cache_with_key_file ] + publish_external " _check_tag_exists() { @@ -150,6 +151,30 @@ cmd_publish() { fi } +cmd_publish_external() { + local image_name=$1; shift || error "Missing target (image name) $USAGE" + # Get version from the command + local image_version=$1; shift || error "Missing target (image version) $USAGE" + + echo "image $image_name:$image_version" + + echo "Publishing and writing to spec cache." + # publish spec to cache. do so, by running get spec locally and then pushing it to gcs. + local tmp_spec_file; tmp_spec_file=$(mktemp) + docker run --rm "$image_name:$image_version" spec | \ + # 1. filter out any lines that are not valid json. + jq -R "fromjson? | ." | \ + # 2. grab any json that has a spec in it. + # 3. if there are more than one, take the first one. + # 4. if there are none, throw an error. + jq -s "map(select(.spec != null)) | map(.spec) | first | if . != null then . else error(\"no spec found\") end" \ + > "$tmp_spec_file" + + echo "Using environment gcloud" + + gsutil cp "$tmp_spec_file" gs://io-airbyte-cloud-spec-cache/specs/"$image_name"/"$image_version"/spec.json +} + main() { assert_root diff --git a/tools/python/.flake8 b/tools/python/.flake8 index b07e01f847baa..ebfec9674bab7 100644 --- a/tools/python/.flake8 +++ b/tools/python/.flake8 @@ -2,6 +2,7 @@ exclude = .venv, models # generated protocol models + airbyte-cdk/python/airbyte_cdk/models/__init__.py .eggs # python libraries" .tox build