diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 65df21860e0807..7cbf98e4587d74 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.36.5-alpha +current_version = 0.36.6-alpha commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index 12df5d2e93b97a..8155983831749a 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.36.5-alpha +VERSION=0.36.6-alpha # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/.github/workflows/build-connector-command.yml b/.github/workflows/build-connector-command.yml new file mode 100644 index 00000000000000..ff39163501bed8 --- /dev/null +++ b/.github/workflows/build-connector-command.yml @@ -0,0 +1,260 @@ +name: Bump, Build, Test Connectors [EXPERIMENTAL] +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + connector: + description: "Airbyte Connector" + required: true + bump-version: + description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually" + required: false + default: "false" + run-tests: + description: "Should run tests" + required: false + default: "true" + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + +jobs: + find_valid_pat: + name: "Find a PAT with room for actions" + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + pat: ${{ steps.variables.outputs.pat }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + id: variables + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + start-bump-build-test-connector-runner: + name: Start Build EC2 Runner + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + # 80 gb disk + ec2-image-id: ami-0d648081937c75a73 + bump-build-test-connector: + name: Bump, Build, Test Connector + needs: start-bump-build-test-connector-runner + runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} + environment: more-secrets + steps: + ############################ + ## SET UP ## + ############################ + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Search for valid connector name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + regex_flags: "i" # required to be set for this plugin + search_string: ${{ github.event.inputs.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != github.event.inputs.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.OCTAVIA_PAT }} + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: "17" + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install Pyenv and Tox + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Install yq + if: github.event.inputs.bump-version != 'false' && success() + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + sudo add-apt-repository ppa:rmescandon/yq + sudo apt update + sudo apt install yq -y + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Get Credentials for ${{ github.event.inputs.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ github.event.inputs.connector }} + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + # TODO: seems like this should run in post-merge workflow + # - name: Prepare Sentry + # if: startsWith(github.event.inputs.connector, 'connectors') + # run: | + # curl -sL https://sentry.io/get-cli/ | bash + # - name: Create Sentry Release + # if: startsWith(github.event.inputs.connector, 'connectors') + # run: | + # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + ############################ + ## BUMP ## + ############################ + - name: Bump Connector Version + if: github.event.inputs.bump-version != 'false' && success() + run: ./tools/integrations/manage.sh bump_version airbyte-integrations/${{ github.event.inputs.connector }} + - name: Commit and Push Version Bump + if: github.event.inputs.bump-version != 'false' && success() + run: | + git config user.name 'Octavia Squidington III' + git config user.email 'octavia-squidington-iii@users.noreply.github.com' + git add -u + git commit -m "bump-version ${{github.event.inputs.connector}}" + git push origin ${{ github.event.inputs.gitref }} + - name: Add Version Bump Success Comment + if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Bumped version for ${{github.event.inputs.connector}} + - name: Add Version Bump Failure Comment + if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Couldn't bump version for ${{github.event.inputs.connector}} + ############################ + ## BUILD AND TEST ## + ############################ + - name: Build ${{ github.event.inputs.connector }} + run: ./tools/integrations/manage.sh build_experiment airbyte-integrations/${{ github.event.inputs.connector }} + id: build + env: + PR_NUMBER: ${{ github.event.number }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + # - name: Test ${{ github.event.inputs.connector }} + # if: github.event.inputs.run-tests == 'true' + # run: ./tools/integrations/manage.sh test airbyte-integrations/${{ github.event.inputs.connector }} + # - name: Finalize Sentry release + # if: startsWith(github.event.inputs.connector, 'connectors') + # run: | + # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Build and Test Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Successfully built and tested ${{github.event.inputs.connector}} + # - name: Build and Test Failure Comment + # if: github.event.inputs.comment-id && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Failed to build and test ${{github.event.inputs.connector}} + # - name: Slack Notification - Failure + # if: failure() + # uses: rtCamp/action-slack-notify@master + # env: + # SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + # SLACK_USERNAME: Buildozer + # SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg + # SLACK_COLOR: DC143C + # SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}" + # SLACK_FOOTER: "" + # - name: Add Final Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # - name: Set publish label + # if: success() + # run: | + # echo "set some label on PR" + # In case of self-hosted EC2 errors, remove this block. + stop-bump-build-test-connector-runner: + name: Stop Build EC2 Runner + needs: + - start-bump-build-test-connector-runner # required to get output from the start-runner job + - bump-build-test-connector # required to wait when the main job is done + - find_valid_pat + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: supertopher/ec2-github-runner@base64v1.0.10 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish-connector-command.yml b/.github/workflows/publish-connector-command.yml new file mode 100644 index 00000000000000..4aac116c8dbfba --- /dev/null +++ b/.github/workflows/publish-connector-command.yml @@ -0,0 +1,210 @@ +name: Publish Connector [EXPERIMENTAL] +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + connector: + description: "Airbyte Connector" + required: true + bump-version: + description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually" + required: false + default: "false" + run-tests: + description: "Should run tests" + required: false + default: "true" + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + +jobs: + find_valid_pat: + name: "Find a PAT with room for actions" + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + pat: ${{ steps.variables.outputs.pat }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + id: variables + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + +# start-bump-build-test-connector-runner: +# name: Start Build EC2 Runner +# runs-on: ubuntu-latest +# needs: find_valid_pat +# outputs: +# label: ${{ steps.start-ec2-runner.outputs.label }} +# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} +# steps: +# - name: Checkout Airbyte +# uses: actions/checkout@v2 +# with: +# repository: ${{ github.event.inputs.repo }} +# ref: ${{ github.event.inputs.gitref }} +# - name: Start AWS Runner +# id: start-ec2-runner +# uses: ./.github/actions/start-aws-runner +# with: +# aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} +# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# # 80 gb disk +# ec2-image-id: ami-0d648081937c75a73 +# bump-build-test-connector: +# needs: start-bump-build-test-connector-runner +# runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} +# environment: more-secrets +# steps: +# ############################ +# ## SET UP ## +# ############################ +# - name: Set up Cloud SDK +# uses: google-github-actions/setup-gcloud@v0 +# with: +# service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} +# export_default_credentials: true +# - name: Search for valid connector name format +# id: regex +# uses: AsasInnab/regex-action@v1 +# with: +# regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" +# regex_flags: "i" # required to be set for this plugin +# search_string: ${{ github.event.inputs.connector }} +# - name: Validate input workflow format +# if: steps.regex.outputs.first_match != github.event.inputs.connector +# run: echo "The connector provided has an invalid format!" && exit 1 +# - name: Link comment to workflow run +# if: github.event.inputs.comment-id +# uses: peter-evans/create-or-update-comment@v1 +# with: +# comment-id: ${{ github.event.inputs.comment-id }} +# body: | +# > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +# - name: Checkout Airbyte +# uses: actions/checkout@v2 +# with: +# repository: ${{ github.event.inputs.repo }} +# ref: ${{ github.event.inputs.gitref }} +# token: ${{ secrets.OCTAVIA_PAT }} +# - name: Install Java +# uses: actions/setup-java@v1 +# with: +# java-version: "17" +# - name: Install Python +# uses: actions/setup-python@v2 +# with: +# python-version: "3.9" +# - name: Install Pyenv and Tox +# run: | +# python3 -m pip install --quiet virtualenv==16.7.9 --user +# python3 -m virtualenv venv +# source venv/bin/activate +# pip install --quiet tox==3.24.4 +# - name: Install yq +# if: github.event.inputs.bump-version != 'false' && success() +# run: | +# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 +# sudo add-apt-repository ppa:rmescandon/yq +# sudo apt update +# sudo apt install yq -y +# - name: Test and install CI scripts +# # all CI python packages have the prefix "ci_" +# run: | +# source venv/bin/activate +# tox -r -c ./tools/tox_ci.ini +# pip install --quiet -e ./tools/ci_* +# - name: Get Credentials for ${{ github.event.inputs.connector }} +# run: | +# source venv/bin/activate +# ci_credentials ${{ github.event.inputs.connector }} +# env: +# GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} +# # TODO: seems like this should run in post-merge workflow +# # - name: Prepare Sentry +# # if: startsWith(github.event.inputs.connector, 'connectors') +# # run: | +# # curl -sL https://sentry.io/get-cli/ | bash +# # - name: Create Sentry Release +# # if: startsWith(github.event.inputs.connector, 'connectors') +# # run: | +# # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing +# # env: +# # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} +# # SENTRY_ORG: airbyte-5j +# # SENTRY_PROJECT: airbyte-connectors +# # - name: Build and Test Success Comment +# # if: github.event.inputs.comment-id && success() +# # uses: peter-evans/create-or-update-comment@v1 +# # with: +# # comment-id: ${{ github.event.inputs.comment-id }} +# # body: | +# # > :rocket: Successfully built and tested ${{github.event.inputs.connector}} +# # - name: Build and Test Failure Comment +# # if: github.event.inputs.comment-id && !success() +# # uses: peter-evans/create-or-update-comment@v1 +# # with: +# # comment-id: ${{ github.event.inputs.comment-id }} +# # body: | +# # > :x: Failed to build and test ${{github.event.inputs.connector}} +# # - name: Slack Notification - Failure +# # if: failure() +# # uses: rtCamp/action-slack-notify@master +# # env: +# # SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} +# # SLACK_USERNAME: Buildozer +# # SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg +# # SLACK_COLOR: DC143C +# # SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}" +# # SLACK_FOOTER: "" +# # - name: Add Final Success Comment +# # if: github.event.inputs.comment-id && success() +# # uses: peter-evans/create-or-update-comment@v1 +# # with: +# # comment-id: ${{ github.event.inputs.comment-id }} +# # body: | +# # > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +# # - name: Set publish label +# # if: success() +# # run: | +# # echo "set some label on PR" +# # In case of self-hosted EC2 errors, remove this block. +# stop-bump-build-test-connector-runner: +# name: Stop Build EC2 Runner +# needs: +# - start-bump-build-test-connector-runner # required to get output from the start-runner job +# - bump-build-test-connector # required to wait when the main job is done +# - find_valid_pat +# runs-on: ubuntu-latest +# if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs +# steps: +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} +# aws-region: us-east-2 +# - name: Stop EC2 runner +# uses: supertopher/ec2-github-runner@base64v1.0.10 +# with: +# mode: stop +# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} +# ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/shared-issues.yml b/.github/workflows/shared-issues.yml new file mode 100644 index 00000000000000..03bbb8eb0554f6 --- /dev/null +++ b/.github/workflows/shared-issues.yml @@ -0,0 +1,16 @@ +name: "Shared Issues" +on: + issues: + types: [opened, labeled, unlabeled] + +jobs: + shared-issues: + runs-on: ubuntu-latest + steps: + - uses: nick-fields/private-action-loader@v3 + with: + pal-repo-token: "${{ secrets.OCTAVIA_PAT }}" + pal-repo-name: airbytehq/workflow-actions@production + # the following input gets passed to the private action + token: "${{ secrets.OCTAVIA_PAT }}" + command: "issue" diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index c1fdc0044d4bc6..c116d5e8bb8ef5 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2323,6 +2323,7 @@ components: type: string CustomerioNotificationConfiguration: type: object + nullable: true NotificationType: type: string enum: @@ -3506,6 +3507,7 @@ components: #- unnesting OperatorDbt: type: object + nullable: true required: - gitRepoUrl properties: @@ -3780,6 +3782,7 @@ components: $ref: "#/components/schemas/AttemptFailureSummary" AttemptStats: type: object + nullable: true properties: recordsEmitted: type: integer @@ -3805,6 +3808,7 @@ components: $ref: "#/components/schemas/AttemptStats" AttemptFailureSummary: type: object + nullable: true required: - failures properties: @@ -4015,6 +4019,7 @@ components: ResourceRequirements: description: optional resource requirements to run workers (blank for unbounded allocations) type: object + nullable: true properties: cpu_request: type: string diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index ddca518b1f5fa0..74e2c9d0c95932 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -1,7 +1,7 @@ ARG JDK_VERSION=17.0.1 FROM openjdk:${JDK_VERSION}-slim -ARG VERSION=0.36.5-alpha +ARG VERSION=0.36.6-alpha ENV APPLICATION airbyte-bootloader ENV VERSION ${VERSION} diff --git a/airbyte-cdk/python/docs/tutorials/http_api_source_assets/configured_catalog.json b/airbyte-cdk/python/docs/tutorials/http_api_source_assets/configured_catalog.json index 66ab9be9e7bb52..7aa9a7e9b2229c 100644 --- a/airbyte-cdk/python/docs/tutorials/http_api_source_assets/configured_catalog.json +++ b/airbyte-cdk/python/docs/tutorials/http_api_source_assets/configured_catalog.json @@ -7,6 +7,9 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { + "access_key": { + "type": "string" + }, "base": { "type": "string" }, diff --git a/airbyte-cdk/python/docs/tutorials/http_api_source_assets/exchange_rates.json b/airbyte-cdk/python/docs/tutorials/http_api_source_assets/exchange_rates.json index 7476b088094e2d..9462ce0079e6ef 100644 --- a/airbyte-cdk/python/docs/tutorials/http_api_source_assets/exchange_rates.json +++ b/airbyte-cdk/python/docs/tutorials/http_api_source_assets/exchange_rates.json @@ -2,6 +2,9 @@ "type": "object", "required": ["base", "date", "rates"], "properties": { + "access_key": { + "type": "string" + }, "base": { "type": "string" }, diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e9f195a598225e..0775676acb8eda 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -24,7 +24,7 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 1.1.1 + dockerImageTag: 1.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg resourceRequirements: @@ -36,7 +36,7 @@ - name: BigQuery (denormalized typed struct) destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-bigquery-denormalized - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index dfa8a93846274e..7eb63ea62bae70 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -285,7 +285,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-bigquery:1.1.1" +- dockerImage: "airbyte/destination-bigquery:1.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: @@ -294,6 +294,7 @@ type: "object" required: - "project_id" + - "dataset_location" - "dataset_id" additionalProperties: true properties: @@ -317,20 +318,12 @@ \ dataset. Read more here." title: "Project ID" - dataset_id: - type: "string" - description: "The default BigQuery Dataset ID that tables are replicated\ - \ to if the source does not specify a namespace. Read more here." - title: "Default Dataset ID" dataset_location: type: "string" description: "The location of the dataset. Warning: Changes made after creation\ - \ will not be applied. The default \"US\" value is used if not set explicitly.\ - \ Read more here." - title: "Dataset Location (Optional)" - default: "US" + title: "Dataset Location" enum: - "US" - "EU" @@ -363,6 +356,12 @@ - "us-west2" - "us-west3" - "us-west4" + dataset_id: + type: "string" + description: "The default BigQuery Dataset ID that tables are replicated\ + \ to if the source does not specify a namespace. Read more here." + title: "Default Dataset ID" credentials_json: type: "string" description: "The contents of the JSON service account key. Check out the\ @@ -495,7 +494,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.1" +- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7a6d3fba64f361..296d70636390b0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -147,7 +147,7 @@ - name: Cockroachdb sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 dockerRepository: airbyte/source-cockroachdb - dockerImageTag: 0.1.11 + dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb icon: cockroachdb.svg sourceType: database @@ -265,7 +265,7 @@ - name: Gitlab sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 dockerRepository: airbyte/source-gitlab - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/gitlab icon: gitlab.svg sourceType: api @@ -348,7 +348,7 @@ - name: IBM Db2 sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 dockerRepository: airbyte/source-db2 - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/db2 icon: db2.svg sourceType: database @@ -362,7 +362,7 @@ - name: Intercom sourceDefinitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a dockerRepository: airbyte/source-intercom - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.17 documentationUrl: https://docs.airbyte.io/integrations/sources/intercom icon: intercom.svg sourceType: api @@ -459,7 +459,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.3.21 + dockerImageTag: 0.3.22 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -473,7 +473,7 @@ - name: Mixpanel sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a dockerRepository: airbyte/source-mixpanel - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel icon: mixpanel.svg sourceType: api @@ -501,7 +501,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 0.5.9 + dockerImageTag: 0.5.10 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -535,7 +535,7 @@ - name: Oracle DB sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 dockerRepository: airbyte/source-oracle - dockerImageTag: 0.3.14 + dockerImageTag: 0.3.15 documentationUrl: https://docs.airbyte.io/integrations/sources/oracle icon: oracle.svg sourceType: database @@ -625,7 +625,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 0.4.11 + dockerImageTag: 0.4.12 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -667,7 +667,7 @@ - name: Redshift sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 dockerRepository: airbyte/source-redshift - dockerImageTag: 0.3.9 + dockerImageTag: 0.3.10 documentationUrl: https://docs.airbyte.io/integrations/sources/redshift icon: redshift.svg sourceType: database @@ -737,7 +737,7 @@ - name: Smartsheets sourceDefinitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc dockerRepository: airbyte/source-smartsheets - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/smartsheets icon: smartsheet.svg sourceType: api @@ -751,7 +751,7 @@ - name: Snowflake sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 dockerRepository: airbyte/source-snowflake - dockerImageTag: 0.1.11 + dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake icon: snowflake.svg sourceType: database @@ -792,14 +792,14 @@ - name: TiDB sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae dockerRepository: airbyte/source-tidb - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/tidb icon: tidb.svg sourceType: database - name: TikTok Marketing sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 dockerRepository: airbyte/source-tiktok-marketing - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing icon: tiktok.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 295060f93df643..9f925c519434d0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1305,7 +1305,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-cockroachdb:0.1.11" +- dockerImage: "airbyte/source-cockroachdb:0.1.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/cockroachdb" connectionSpecification: @@ -2595,7 +2595,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-gitlab:0.1.4" +- dockerImage: "airbyte/source-gitlab:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/gitlab" connectionSpecification: @@ -3578,7 +3578,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-db2:0.1.9" +- dockerImage: "airbyte/source-db2:0.1.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/db2" connectionSpecification: @@ -3709,7 +3709,7 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-intercom:0.1.16" +- dockerImage: "airbyte/source-intercom:0.1.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/intercom" connectionSpecification: @@ -4669,7 +4669,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.3.21" +- dockerImage: "airbyte/source-mssql:0.3.22" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -4687,6 +4687,7 @@ description: "The hostname of the database." title: "Host" type: "string" + order: 0 port: description: "The port of the database." title: "Port" @@ -4695,21 +4696,32 @@ maximum: 65536 examples: - "1433" + order: 1 database: description: "The name of the database." title: "Database" type: "string" examples: - "master" + order: 2 username: description: "The username which is used to access the database." title: "Username" type: "string" + order: 3 password: description: "The password associated with the username." title: "Password" type: "string" airbyte_secret: true + order: 4 + jdbc_url_params: + title: "JDBC URL Params" + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + type: "string" + order: 5 ssl_method: title: "SSL Method" type: "object" @@ -4774,6 +4786,7 @@ enum: - "STANDARD" - "CDC" + order: 8 tunnel_method: type: "object" title: "SSH Tunnel Method" @@ -5025,7 +5038,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-mixpanel:0.1.13" +- dockerImage: "airbyte/source-mixpanel:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel" connectionSpecification: @@ -5368,7 +5381,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.5.9" +- dockerImage: "airbyte/source-mysql:0.5.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" connectionSpecification: @@ -5757,7 +5770,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-oracle:0.3.14" +- dockerImage: "airbyte/source-oracle:0.3.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/oracle" connectionSpecification: @@ -6478,7 +6491,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:0.4.11" +- dockerImage: "airbyte/source-postgres:0.4.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -6914,7 +6927,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-redshift:0.3.9" +- dockerImage: "airbyte/source-redshift:0.3.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" connectionSpecification: @@ -7861,7 +7874,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-smartsheets:0.1.10" +- dockerImage: "airbyte/source-smartsheets:0.1.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/smartsheets" connectionSpecification: @@ -7878,13 +7891,15 @@ description: "The Access Token for making authenticated requests. Find in\ \ the main menu: Account > Apps & Integrations > API Access" type: "string" + order: 0 airbyte_secret: true spreadsheet_id: title: "Sheet ID" description: "The spreadsheet ID. Find in the spreadsheet menu: File > Properties" type: "string" + order: 1 start_datetime: - title: "Start Datetime" + title: "Start Datetime (Optional)" type: "string" examples: - "2000-01-01T13:00:00" @@ -7892,6 +7907,7 @@ description: "ISO 8601, for instance: `YYYY-MM-DDTHH:MM:SS`, `YYYY-MM-DDTHH:MM:SS+HH:MM`" format: "date-time" default: "2020-01-01T00:00:00+00:00" + order: 2 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] @@ -7969,7 +7985,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-snowflake:0.1.11" +- dockerImage: "airbyte/source-snowflake:0.1.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake" connectionSpecification: @@ -8490,7 +8506,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tidb:0.1.0" +- dockerImage: "airbyte/source-tidb:0.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/tidb" connectionSpecification: @@ -8650,7 +8666,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tiktok-marketing:0.1.7" +- dockerImage: "airbyte/source-tiktok-marketing:0.1.8" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing" @@ -8684,7 +8700,7 @@ type: "string" access_token: title: "Access Token" - description: "Long-term Authorized Access Token." + description: "The long-term authorized access token." airbyte_secret: true type: "string" required: @@ -8710,7 +8726,7 @@ type: "string" access_token: title: "Access Token" - description: "The Long-term Authorized Access Token." + description: "The long-term authorized access token." airbyte_secret: true type: "string" required: @@ -8727,12 +8743,12 @@ type: "string" advertiser_id: title: "Advertiser ID" - description: "The Advertiser ID which generated for the developer's\ + description: "The Advertiser ID which generated for the developer's\ \ Sandbox application." type: "string" access_token: title: "Access Token" - description: "The Long-term Authorized Access Token." + description: "The long-term authorized access token." airbyte_secret: true type: "string" required: @@ -8741,17 +8757,16 @@ start_date: title: "Start Date *" description: "The Start Date in format: YYYY-MM-DD. Any data before this\ - \ date will not be replicated. If this parameter is not set, all data\ - \ will be replicated." + \ date will not be replicated.If this parameter is not set, all data will\ + \ be replicated." default: "2016-09-01" pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" order: 1 type: "string" report_granularity: title: "Report Granularity *" - description: "Which time granularity should be grouped by; for LIFETIME\ - \ there will be no grouping. This option is used for reports' streams\ - \ only." + description: "Grouping of your reports based on time. Lifetime will have\ + \ no grouping. This option is used for reports' streams only." default: "DAY" enum: - "LIFETIME" diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 159cf08bae891a..4eb292021a6bdf 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -25,7 +25,7 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list RUN apt-get update && apt-get install -y kubectl -ARG VERSION=0.36.5-alpha +ARG VERSION=0.36.6-alpha ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 8c5d69903d89c0..3d52778abfcf67 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -4,6 +4,7 @@ plugins { dependencies { api 'org.apache.commons:commons-dbcp2:2.7.0' + api 'com.zaxxer:HikariCP:5.0.1' api 'org.jooq:jooq-meta:3.13.4' api 'org.jooq:jooq:3.13.4' api 'org.postgresql:postgresql:42.2.18' diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index 1e193409e042ec..2df0b624f0794a 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -24,6 +24,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Provides utility methods to create configured {@link Database} instances. + * + * @deprecated This class has been marked as deprecated as we move to using an application framework + * to manage resources. This class will be removed in a future release. + * + * @see io.airbyte.db.factory.DataSourceFactory + * @see io.airbyte.db.factory.DSLContextFactory + * @see io.airbyte.db.factory.FlywayFactory + */ +@Deprecated(forRemoval = true) public class Databases { private static final Logger LOGGER = LoggerFactory.getLogger(Databases.class); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java new file mode 100644 index 00000000000000..eba32e7cb62017 --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; + +/** + * Temporary factory class that provides convenience methods for creating a {@link DSLContext} + * instances. This class will be removed once the project has been converted to leverage an + * application framework to manage the creation and injection of {@link DSLContext} objects. + * + * This class replaces direct calls to {@link io.airbyte.db.Databases}. + */ +public class DSLContextFactory { + + /** + * Constructs a configured {@link DSLContext} instance using the provided configuration. + * + * @param dataSource The {@link DataSource} used to connect to the database. + * @param dialect The SQL dialect to use with objects created from this context. + * @return The configured {@link DSLContext}. + */ + public static DSLContext create(final DataSource dataSource, final SQLDialect dialect) { + return DSL.using(dataSource, dialect); + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java new file mode 100644 index 00000000000000..31c659a548cfb3 --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.util.Map; +import javax.sql.DataSource; + +/** + * Temporary factory class that provides convenience methods for creating a {@link DataSource} + * instance. This class will be removed once the project has been converted to leverage an + * application framework to manage the creation and injection of {@link DataSource} objects. + * + * This class replaces direct calls to {@link io.airbyte.db.Databases}. + */ +public class DataSourceFactory { + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString) { + return new DataSourceBuilder() + .withDriverClassName(driverClassName) + .withJdbcUrl(jdbcConnectionString) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @param connectionProperties Additional configuration properties for the underlying driver. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString, + final Map connectionProperties) { + return new DataSourceBuilder() + .withConnectionProperties(connectionProperties) + .withDriverClassName(driverClassName) + .withJdbcUrl(jdbcConnectionString) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String host, + final int port, + final String database, + final String driverClassName) { + return new DataSourceBuilder() + .withDatabase(database) + .withDriverClassName(driverClassName) + .withHost(host) + .withPort(port) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param connectionProperties Additional configuration properties for the underlying driver. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String host, + final int port, + final String database, + final String driverClassName, + final Map connectionProperties) { + return new DataSourceBuilder() + .withConnectionProperties(connectionProperties) + .withDatabase(database) + .withDriverClassName(driverClassName) + .withHost(host) + .withPort(port) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Convenience method that constructs a new {@link DataSource} for a PostgreSQL database using the + * provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @return The configured {@link DataSource}. + */ + public static DataSource createPostgres(final String username, + final String password, + final String host, + final int port, + final String database) { + return new DataSourceBuilder() + .withDatabase(database) + .withDriverClassName("org.postgresql.Driver") + .withHost(host) + .withPort(port) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Builder class used to configure and construct {@link DataSource} instances. + */ + private static class DataSourceBuilder { + + private static final Map JDBC_URL_FORMATS = Map.of("org.postgresql.Driver", "jdbc:postgresql://%s:%d/%s", + "com.amazon.redshift.jdbc.Driver", "jdbc:redshift://%s:%d/%s", + "com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%d/%s", + "com.microsoft.sqlserver.jdbc.SQLServerDriver", "jdbc:sqlserver://%s:%d/%s", + "oracle.jdbc.OracleDriver", "jdbc:oracle:thin:@%s:%d:%s", + "ru.yandex.clickhouse.ClickHouseDriver", "jdbc:ch://%s:%d/%s", + "org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"); + + private Map connectionProperties = Map.of(); + private String database; + private String driverClassName = "org.postgresql.Driver"; + private String host; + private String jdbcUrl; + private Integer maximumPoolSize = 5; + private Integer minimumPoolSize = 0; + private String password; + private Integer port = 5432; + private String username; + + private DataSourceBuilder() {} + + public DataSourceBuilder withConnectionProperties(final Map connectionProperties) { + if (connectionProperties != null) { + this.connectionProperties = connectionProperties; + } + return this; + } + + public DataSourceBuilder withDatabase(final String database) { + this.database = database; + return this; + } + + public DataSourceBuilder withDriverClassName(final String driverClassName) { + this.driverClassName = driverClassName; + return this; + } + + public DataSourceBuilder withHost(final String host) { + this.host = host; + return this; + } + + public DataSourceBuilder withJdbcUrl(final String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + return this; + } + + public DataSourceBuilder withMaximumPoolSize(final Integer maximumPoolSize) { + if (maximumPoolSize != null) { + this.maximumPoolSize = maximumPoolSize; + } + return this; + } + + public DataSourceBuilder withMinimumPoolSize(final Integer minimumPoolSize) { + if (minimumPoolSize != null) { + this.minimumPoolSize = minimumPoolSize; + } + return this; + } + + public DataSourceBuilder withPassword(final String password) { + this.password = password; + return this; + } + + public DataSourceBuilder withPort(final Integer port) { + if (port != null) { + this.port = port; + } + return this; + } + + public DataSourceBuilder withUsername(final String username) { + this.username = username; + return this; + } + + public DataSource build() { + final HikariConfig config = new HikariConfig(); + config.setDriverClassName(driverClassName); + config.setJdbcUrl(jdbcUrl != null ? jdbcUrl : String.format(JDBC_URL_FORMATS.getOrDefault(driverClassName, ""), host, port, database)); + config.setMaximumPoolSize(maximumPoolSize); + config.setMinimumIdle(minimumPoolSize); + config.setPassword(password); + config.setUsername(username); + + connectionProperties.forEach(config::addDataSourceProperty); + + final HikariDataSource dataSource = new HikariDataSource(config); + dataSource.validate(); + return dataSource; + } + + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java new file mode 100644 index 00000000000000..0e5526745fd94d --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; + +/** + * Temporary factory class that provides convenience methods for creating a {@link Flyway} + * instances. This class will be removed once the project has been converted to leverage an + * application framework to manage the creation and injection of {@link Flyway} objects. + * + * This class replaces direct calls to {@link io.airbyte.db.Databases}. + */ +public class FlywayFactory { + + static final String MIGRATION_TABLE_FORMAT = "airbyte_%s_migrations"; + + // Constants for Flyway baseline. See here for details: + // https://flywaydb.org/documentation/command/baseline + static final String BASELINE_VERSION = "0.29.0.001"; + static final String BASELINE_DESCRIPTION = "Baseline from file-based migration v1"; + static final boolean BASELINE_ON_MIGRATION = true; + + /** + * Constructs a configured {@link Flyway} instance using the provided configuration. + * + * @param dataSource The {@link DataSource} used to connect to the database. + * @param installedBy The name of the module performing the migration. + * @param dbIdentifier The name of the database to be migrated. This is used to name the table to + * hold the migration history for the database. + * @param migrationFileLocations The array of migration files to be used. + * @return The configured {@link Flyway} instance. + */ + public static Flyway create(final DataSource dataSource, + final String installedBy, + final String dbIdentifier, + final String... migrationFileLocations) { + return Flyway.configure() + .dataSource(dataSource) + .baselineVersion(BASELINE_VERSION) + .baselineDescription(BASELINE_DESCRIPTION) + .baselineOnMigrate(BASELINE_ON_MIGRATION) + .installedBy(installedBy) + .table(String.format(MIGRATION_TABLE_FORMAT, dbIdentifier)) + .locations(migrationFileLocations) + .load(); + + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 05caf59336b02d..50471c0da2034e 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -29,17 +29,25 @@ public static String getFullyQualifiedTableName(final String schemaName, final S } public static Map parseJdbcParameters(final JsonNode config, final String jdbcUrlParamsKey) { + return parseJdbcParameters(config, jdbcUrlParamsKey, "&"); + } + + public static Map parseJdbcParameters(final JsonNode config, final String jdbcUrlParamsKey, final String delimiter) { if (config.has(jdbcUrlParamsKey)) { - return parseJdbcParameters(config.get(jdbcUrlParamsKey).asText()); + return parseJdbcParameters(config.get(jdbcUrlParamsKey).asText(), delimiter); } else { return Maps.newHashMap(); } } public static Map parseJdbcParameters(final String jdbcPropertiesString) { + return parseJdbcParameters(jdbcPropertiesString, "&"); + } + + public static Map parseJdbcParameters(final String jdbcPropertiesString, final String delimiter) { final Map parameters = new HashMap<>(); if (!jdbcPropertiesString.isBlank()) { - final String[] keyValuePairs = jdbcPropertiesString.split("&"); + final String[] keyValuePairs = jdbcPropertiesString.split(delimiter); for (final String kv : keyValuePairs) { final String[] split = kv.split("="); if (split.length == 2) { diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/AbstractFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/AbstractFactoryTest.java new file mode 100644 index 00000000000000..25f8b4c4ca3ed8 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/AbstractFactoryTest.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * Common test suite for the classes found in the {@code io.airbyte.db.factory} package. + */ +public abstract class AbstractFactoryTest { + + private static final String DATABASE_NAME = "airbyte_test_database"; + + protected static PostgreSQLContainer container; + + @BeforeAll + public static void dbSetup() { + container = new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName(DATABASE_NAME) + .withUsername("docker") + .withPassword("docker"); + container.start(); + } + + @AfterAll + public static void dbDown() { + container.close(); + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java new file mode 100644 index 00000000000000..b4bae85c24f921 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.Test; +import org.postgresql.Driver; + +/** + * Test suite for the {@link DSLContextFactory} class. + */ +public class DSLContextFactoryTest extends AbstractFactoryTest { + + @Test + void testCreatingADslContext() { + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), Driver.class.getName(), container.getJdbcUrl()); + final SQLDialect dialect = SQLDialect.POSTGRES; + final DSLContext dslContext = DSLContextFactory.create(dataSource, dialect); + assertNotNull(dslContext); + assertEquals(dialect, dslContext.configuration().dialect()); + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java new file mode 100644 index 00000000000000..4cfe7cc1412453 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.zaxxer.hikari.HikariDataSource; +import java.util.Map; +import javax.sql.DataSource; +import org.junit.jupiter.api.Test; +import org.postgresql.Driver; + +/** + * Test suite for the {@link DataSourceFactory} class. + */ +public class DataSourceFactoryTest extends AbstractFactoryTest { + + @Test + void testCreatingADataSourceWithJdbcUrl() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String jdbcUrl = container.getJdbcUrl(); + + final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingADataSourceWithJdbcUrlAndConnectionProperties() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String jdbcUrl = container.getJdbcUrl(); + final Map connectionProperties = Map.of("foo", "bar"); + + final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl, connectionProperties); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingADataSourceWithHostAndPort() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + + final DataSource dataSource = DataSourceFactory.create(username, password, host, port, database, driverClassName); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingADataSourceWithHostPortAndConnectionProperties() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + final Map connectionProperties = Map.of("foo", "bar"); + + final DataSource dataSource = DataSourceFactory.create(username, password, host, port, database, driverClassName, connectionProperties); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingAnInvalidDataSourceWithHostAndPort() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = "Unknown"; + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + + assertThrows(RuntimeException.class, () -> { + DataSourceFactory.create(username, password, host, port, database, driverClassName); + }); + } + + @Test + void testCreatingAPostgresqlDataSource() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + + final DataSource dataSource = DataSourceFactory.createPostgres(username, password, host, port, database); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java new file mode 100644 index 00000000000000..2c2913261b2839 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.postgresql.Driver; + +/** + * Test suite for the {@link FlywayFactory} class. + */ +public class FlywayFactoryTest extends AbstractFactoryTest { + + @Test + void testCreatingAFlywayInstance() { + final String installedBy = "test"; + final String dbIdentifier = "test"; + final String migrationFileLocation = "classpath:io/airbyte/db/instance/toys/migrations"; + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), Driver.class.getName(), container.getJdbcUrl()); + + final Flyway flyway = FlywayFactory.create(dataSource, installedBy, dbIdentifier, migrationFileLocation); + assertNotNull(flyway); + assertTrue(flyway.getConfiguration().isBaselineOnMigrate()); + assertEquals(FlywayFactory.BASELINE_DESCRIPTION, flyway.getConfiguration().getBaselineDescription()); + assertEquals(FlywayFactory.BASELINE_VERSION, flyway.getConfiguration().getBaselineVersion().getVersion()); + assertEquals(installedBy, flyway.getConfiguration().getInstalledBy()); + assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, dbIdentifier), flyway.getConfiguration().getTable()); + assertEquals(migrationFileLocation, flyway.getConfiguration().getLocations()[0].getDescriptor()); + } + +} diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 62bfda98ba17ff..d39bf471be5840 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -135,6 +135,8 @@ | MariaDB ColumnStore | [![destination-mariadb-columnstore](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-mariadb-columnstore%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-mariadb-columnstore) | | Mongo DB | [![destination-mongodb](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-mongodb%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-mongodb) | | MQTT | [![destination-mqtt](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-mqtt%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-mqtt) | +| MSSQL (SQL Server) | [![destination-mssql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-mssql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-mssql) | +| MySQL | [![destination-mysql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-mysql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-mysql) | | Postgres | [![destination-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-postgres%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-postgres) | | Pulsar | [![destination-pulsar](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-pulsar%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-pulsar) | | Redshift | [![destination-redshift](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-redshift%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-redshift) | diff --git a/airbyte-integrations/connector-templates/generator/package-lock.json b/airbyte-integrations/connector-templates/generator/package-lock.json index 0fb11c7cd76cb5..9a26b438e72cda 100644 --- a/airbyte-integrations/connector-templates/generator/package-lock.json +++ b/airbyte-integrations/connector-templates/generator/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "devDependencies": { "capital-case": "^1.0.4", + "change-case": "^4.1.2", "handlebars": "^4.7.7", "plop": "^3.0.5", "set-value": ">=4.0.1", diff --git a/airbyte-integrations/connector-templates/generator/package.json b/airbyte-integrations/connector-templates/generator/package.json index 0a4ae5ac873f17..f7bd9559da45d3 100644 --- a/airbyte-integrations/connector-templates/generator/package.json +++ b/airbyte-integrations/connector-templates/generator/package.json @@ -7,6 +7,7 @@ }, "devDependencies": { "capital-case": "^1.0.4", + "change-case": "^4.1.2", "handlebars": "^4.7.7", "plop": "^3.0.5", "set-value": ">=4.0.1", diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index 86c5f215755218..d9914238b4a460 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -2,7 +2,8 @@ const path = require('path'); const uuid = require('uuid'); const capitalCase = require('capital-case'); - +const changeCase = require('change-case') + const getSuccessMessage = function(connectorName, outputPath, additionalMessage){ return ` 🚀 🚀 🚀 🚀 🚀 🚀 @@ -27,6 +28,8 @@ module.exports = function (plop) { const docRoot = '../../../docs/integrations'; const definitionRoot = '../../../airbyte-config/init/src/main/resources'; + const sourceAcceptanceTestFilesInputRoot = '../source_acceptance_test_files'; + const pythonSourceInputRoot = '../source-python'; const singerSourceInputRoot = '../source-singer'; const genericSourceInputRoot = '../source-generic'; @@ -43,11 +46,40 @@ module.exports = function (plop) { const httpApiOutputRoot = `${outputDir}/source-{{dashCase name}}`; const javaDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`; const pythonDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`; + const sourceConnectorImagePrefix = 'airbyte/source-' + const sourceConnectorImageTag = 'dev' + const defaultSpecPathFolderPrefix = 'source_' + const specFileName = 'spec.yaml' + plop.setHelper('capitalCase', function(name) { return capitalCase.capitalCase(name); }); + plop.setHelper('connectorImage', function() { + let suffix = "" + if (typeof this.connectorImageNameSuffix !== 'undefined') { + suffix = this.connectorImageNameSuffix + } + return `${sourceConnectorImagePrefix}${changeCase.paramCase(this.name)}${suffix}:${sourceConnectorImageTag}` + }); + + plop.setHelper('specPath', function() { + let suffix = "" + if (typeof this.specPathFolderSuffix !== 'undefined') { + suffix = this.specPathFolderSuffix + } + let inSubFolder = true + if (typeof this.inSubFolder !== 'undefined') { + inSubFolder = this.inSubFolder + } + if (inSubFolder) { + return `${defaultSpecPathFolderPrefix}${changeCase.snakeCase(this.name)}${suffix}/${specFileName}` + } else { + return specFileName + } + }); + plop.setActionType('emitSuccess', function(answers, config, plopApi){ console.log(getSuccessMessage(answers.name, plopApi.renderString(config.outputPath, answers), config.message)); }); @@ -86,6 +118,14 @@ module.exports = function (plop) { base: httpApiInputRoot, templateFiles: `${httpApiInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: httpApiOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + }, // plop doesn't add dotfiles by default so we manually add them { type:'add', @@ -113,6 +153,18 @@ module.exports = function (plop) { base: singerSourceInputRoot, templateFiles: `${singerSourceInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: singerSourceOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + data: { + connectorImageNameSuffix: "-singer", + specPathFolderSuffix: "_singer" + } + }, { type:'add', abortOnFail: true, @@ -140,6 +192,14 @@ module.exports = function (plop) { base: pythonSourceInputRoot, templateFiles: `${pythonSourceInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: pythonSourceOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + }, { type:'add', abortOnFail: true, @@ -175,6 +235,17 @@ module.exports = function (plop) { base: genericSourceInputRoot, templateFiles: `${genericSourceInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: genericSourceOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + data: { + inSubFolder: false + } + }, {type: 'emitSuccess', outputPath: genericSourceOutputRoot} ] }); diff --git a/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml b/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml deleted file mode 100644 index 4d2eeb3e00e6ae..00000000000000 --- a/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml +++ /dev/null @@ -1,25 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}:dev -tests: - spec: - - spec_path: "spec.json" - config_path: "secrets/valid_config.json" # TODO add this file - connection: - - config_path: "secrets/valid_config.json" # TODO add this file - status: "succeed" - - config_path: "secrets/invalid_config.json" # TODO add this file - status: "failed" - discovery: - - config_path: "secrets/valid_config.json" - basic_read: - - config_path: "secrets/valid_config.json" - configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file - empty_streams: [] - full_refresh: - - config_path: "secrets/valid_config.json" - configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file -# incremental: # TODO uncomment this once you implement incremental sync -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs index d7babcc57203b0..3d3350f9a0106c 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs @@ -6,7 +6,7 @@ package io.airbyte.integrations.source.{{snakeCase name}}; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.NoOpStreamingQueryConfig; +import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; @@ -23,9 +23,9 @@ public class {{pascalCase name}}Source extends AbstractJdbcSource impl static final String DRIVER_CLASS = "driver_name_here"; public {{pascalCase name}}Source() { - // By default, NoOpStreamingQueryConfig class is used. If the JDBC supports custom - // fetch size, change it to AdaptiveStreamingQueryConfig for better performance. - super(DRIVER_CLASS, NoOpStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); + // TODO: if the JDBC driver does not support custom fetch size, use NoOpStreamingQueryConfig + // instead of AdaptiveStreamingQueryConfig. + super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); } // TODO The config is based on spec.json, update according to your DB diff --git a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-config.yml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-config.yml.hbs deleted file mode 100644 index 49acab015f3f86..00000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-config.yml.hbs +++ /dev/null @@ -1,30 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}:dev -tests: - spec: - - spec_path: "source_{{snakeCase name}}/spec.yaml" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690c1..00000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connector-templates/source-python/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-python/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690c1..00000000000000 --- a/airbyte-integrations/connector-templates/source-python/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connector-templates/source-singer/acceptance-test-config.yml.hbs b/airbyte-integrations/connector-templates/source-singer/acceptance-test-config.yml.hbs deleted file mode 100644 index f485a8c6460ddb..00000000000000 --- a/airbyte-integrations/connector-templates/source-singer/acceptance-test-config.yml.hbs +++ /dev/null @@ -1,30 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}-singer:dev -tests: - spec: - - spec_path: "source_{{snakeCase name}}_singer/spec.yaml" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "exception" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connector-templates/source-singer/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-singer/acceptance-test-docker.sh deleted file mode 100644 index e4d8b1cef8961e..00000000000000 --- a/airbyte-integrations/connector-templates/source-singer/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connector-templates/source-python/acceptance-test-config.yml.hbs b/airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-config.yml.hbs similarity index 92% rename from airbyte-integrations/connector-templates/source-python/acceptance-test-config.yml.hbs rename to airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-config.yml.hbs index 2ec2ab2694f8f0..3981383e5d7657 100644 --- a/airbyte-integrations/connector-templates/source-python/acceptance-test-config.yml.hbs +++ b/airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-config.yml.hbs @@ -1,9 +1,9 @@ # See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}:dev +connector_image: {{ connectorImage }} tests: spec: - - spec_path: "source_{{snakeCase name}}/spec.yaml" + - spec_path: "{{ specPath }}" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-docker.sh similarity index 100% rename from airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh rename to airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-docker.sh diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 254ccae1e75b82..3ef5865c9d0e1e 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.3 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index ceab2eaaa91069..403fd94981c33b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.1.1 +LABEL io.airbyte.version=1.1.3 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index 3e74bbb81283ab..b51812ee0d7b0e 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -8,7 +8,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "BigQuery Destination Spec", "type": "object", - "required": ["project_id", "dataset_id"], + "required": ["project_id", "dataset_location", "dataset_id"], "additionalProperties": true, "properties": { "big_query_client_buffer_size_mb": { @@ -25,16 +25,10 @@ "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", "title": "Project ID" }, - "dataset_id": { - "type": "string", - "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", - "title": "Default Dataset ID" - }, "dataset_location": { "type": "string", - "description": "The location of the dataset. Warning: Changes made after creation will not be applied. The default \"US\" value is used if not set explicitly. Read more here.", - "title": "Dataset Location (Optional)", - "default": "US", + "description": "The location of the dataset. Warning: Changes made after creation will not be applied. Read more here.", + "title": "Dataset Location", "enum": [ "US", "EU", @@ -69,6 +63,11 @@ "us-west4" ] }, + "dataset_id": { + "type": "string", + "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", + "title": "Default Dataset ID" + }, "credentials_json": { "type": "string", "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index b360964abcc4fb..6cfde5013241d5 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -178,8 +178,7 @@ public void testEncryption() throws SQLException { config.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm + " )")); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); final String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -204,8 +203,7 @@ public void testCheckProtocol() throws SQLException { clone.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm + " )")); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); final String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.unsafeQuery(network_service_banner).collect(Collectors.toList()); diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile index 5e07282f511249..f68558cfd60772 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile index 8a8456ed239de8..8ae1e6922049af 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java index bd2ebe01a3b363..dfe64063cf5923 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java @@ -12,7 +12,7 @@ import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.NoOpStreamingQueryConfig; +import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshWrappedSource; @@ -41,7 +41,7 @@ public class CockroachDbSource extends AbstractJdbcSource { public static final List PORT_KEY = List.of("port"); public CockroachDbSource() { - super(DRIVER_CLASS, NoOpStreamingQueryConfig::new, new CockroachJdbcSourceOperations()); + super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new CockroachJdbcSourceOperations()); } public static Source sshWrappedSource() { diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile index f992a81d9cc05d..dfa81a644c1e6e 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java index 63c23e74177a43..a21c00c2f0100e 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.Db2Container; @@ -181,7 +182,7 @@ private String getCertificate() throws IOException, InterruptedException { private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); - try (final PrintWriter out = new PrintWriter("certificate.pem")) { + try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) { out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java index 0627551bfeb2a1..61f79b0655d996 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.sql.JDBCType; import java.util.Collections; import java.util.Set; @@ -186,7 +187,7 @@ private static String getCertificate() throws IOException, InterruptedException private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); - try (final PrintWriter out = new PrintWriter("certificate.pem")) { + try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) { out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile index f658419be0068c..d2e3e152e7cc96 100644 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ b/airbyte-integrations/connectors/source-db2/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java index 74b897c7d93b21..f48de8c9e2dc1a 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java @@ -16,6 +16,7 @@ import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.JDBCType; import java.sql.PreparedStatement; @@ -143,7 +144,7 @@ private static String getKeyStorePassword(final JsonNode encryptionKeyStorePassw private static void convertAndImportCertificate(final String certificate, final String keyStorePassword) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); - try (final PrintWriter out = new PrintWriter("certificate.pem")) { + try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) { out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java index 3aa28e9dcaf213..f35bbb52e2095c 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.Db2Container; @@ -175,7 +176,7 @@ private String getCertificate() throws IOException, InterruptedException { private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); - try (final PrintWriter out = new PrintWriter("certificate.pem")) { + try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) { out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index 9cd9977769ff10..f831f3760d4c64 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json index b440ceb148bd83..e7b21178e791ca 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json @@ -138,7 +138,7 @@ "type": ["null", "boolean"] }, "container_expiration_policy": { - "type": "object", + "type": ["null", "object"], "properties": { "cadence": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py index d7dea49b254bad..d05a7188838393 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py @@ -43,7 +43,7 @@ def _generate_main_streams(self, config: Mapping[str, Any]) -> Tuple[GitlabStrea auth = TokenAuthenticator(token=config["private_token"]) auth_params = dict(authenticator=auth, api_url=config["api_url"]) - pids = list(filter(None, config.get("projects").split(" "))) + pids = list(filter(None, config.get("projects", "").split(" "))) gids = config.get("groups") if gids: diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 504b0f5d2164d4..c00e724d30857e 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.55 +LABEL io.airbyte.version=0.1.56 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 57221ba6d0dfc8..764618dd229877 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -331,32 +331,41 @@ def read_records( pagination_complete = False next_page_token = None - with AirbyteSentry.start_transaction("read_records", self.name), AirbyteSentry.start_transaction_span("read_records"): - while not pagination_complete: - - properties_list = list(self.properties.keys()) - if properties_list: - stream_records, response = self._read_stream_records( - properties_list=properties_list, - stream_slice=stream_slice, - stream_state=stream_state, - next_page_token=next_page_token, - ) - records = [value for key, value in stream_records.items()] - else: - response = self.handle_request(stream_slice=stream_slice, stream_state=stream_state, next_page_token=next_page_token) - records = self._transform(self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)) - - if self.filter_old_records: - records = self._filter_old_records(records) - yield from records - - next_page_token = self.next_page_token(response) - if not next_page_token: - pagination_complete = True - - # Always return an empty generator just in case no records were ever yielded - yield from [] + try: + with AirbyteSentry.start_transaction("read_records", self.name), AirbyteSentry.start_transaction_span("read_records"): + while not pagination_complete: + + properties_list = list(self.properties.keys()) + if properties_list: + stream_records, response = self._read_stream_records( + properties_list=properties_list, + stream_slice=stream_slice, + stream_state=stream_state, + next_page_token=next_page_token, + ) + records = [value for key, value in stream_records.items()] + else: + response = self.handle_request( + stream_slice=stream_slice, stream_state=stream_state, next_page_token=next_page_token + ) + records = self._transform(self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)) + + if self.filter_old_records: + records = self._filter_old_records(records) + yield from records + + next_page_token = self.next_page_token(response) + if not next_page_token: + pagination_complete = True + + # Always return an empty generator just in case no records were ever yielded + yield from [] + except requests.exceptions.HTTPError as e: + status_code = e.response.status_code + if status_code == 403: + raise RuntimeError(f"Invalid permissions for {self.name}. Please ensure the all scopes are authorized for.") + else: + raise e @staticmethod def _convert_datetime_to_string(dt: pendulum.datetime, declared_format: str = None) -> str: diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile index d409c57f04268c..136e017187579b 100644 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -35,5 +35,5 @@ COPY source_intercom ./source_intercom ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 3b00101bc44f38..446cb45921582e 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -10,6 +10,7 @@ import requests from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -284,6 +285,7 @@ class Conversations(IncrementalIntercomStream): Endpoint: https://api.intercom.io/conversations """ + use_cache = True data_fields = ["conversations"] def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: @@ -302,20 +304,49 @@ def path(self, **kwargs) -> str: return "conversations" -class ConversationParts(ChildStreamMixin, IncrementalIntercomStream): +class ConversationParts(IncrementalIntercomStream): """Return list of all conversation parts. API Docs: https://developers.intercom.com/intercom-api-reference/reference#retrieve-a-conversation Endpoint: https://api.intercom.io/conversations/ """ data_fields = ["conversation_parts", "conversation_parts"] - parent_stream_class = Conversations + + def __init__(self, authenticator: AuthBase, start_date: str = None, **kwargs): + super().__init__(authenticator, start_date, **kwargs) + self.conversations_stream = Conversations(authenticator, start_date, **kwargs) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"/conversations/{stream_slice['id']}" + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + """ + Returns the stream slices, which correspond to conversation IDs. Uses the `Conversations` stream + to get conversations by `sync_mode` and `state`. Unlike `ChildStreamMixin`, it gets slices based + on the `sync_mode`, so that it does not get all conversations at all times. Since we can't do + `filter_by_state` inside `parse_records`, we need to make sure we get the right conversations only. + Otherwise, this stream would always return all conversation_parts. + """ + parent_stream_slices = self.conversations_stream.stream_slices( + sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state + ) + for stream_slice in parent_stream_slices: + conversations = self.conversations_stream.read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + for conversation in conversations: + yield {"id": conversation["id"]} + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - records = super().parse_response(response, stream_state, **kwargs) + """ + Adds `conversation_id` to every `conversation_part` record before yielding it. Records are not + filtered by state here, because the aggregate list of `conversation_parts` is not sorted by + `updated_at`, because it gets `conversation_parts` for each `conversation`. Hence, using parent's + `filter_by_state` logic could potentially end up in data loss. + """ + records = super().parse_response(response=response, stream_state={}, **kwargs) conversation_id = response.json().get("id") for conversation_part in records: conversation_part.setdefault("conversation_id", conversation_id) diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py index 7173f59594755e..21cb1a83b36515 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py @@ -179,7 +179,7 @@ def test_streams(config): (Teams, "/teams", {"teams": [{"type": "team", "id": "id"}]}, [{"id": "id", "type": "team"}]), ], ) -def test_read(stream, endpoint, response, expected, config, requests_mock): +def test_read(stream, endpoint, response, expected, requests_mock): requests_mock.get("/conversations", json=response) requests_mock.get("/companies/scroll", json=response) requests_mock.get(endpoint, json=response) @@ -194,12 +194,29 @@ def test_read(stream, endpoint, response, expected, config, requests_mock): assert records == expected -def test_conversation_part_has_conversation_id(requests_mock): - """ - Test shows that conversation_part records include the `conversation_id` field. - """ +def build_conversations_response_body(conversations, next_url = None): + return { + "type": "conversation.list", + "pages": {"next": next_url} if next_url else {}, + "conversations": conversations + } + + +def build_conversation_response_body(conversation_id, conversation_parts): + return { + "type": "conversation", + "id": conversation_id, + "conversation_parts": { + "type": "conversation_part.list", + "conversation_parts": conversation_parts, + "total_count": len(conversation_parts), + }, + } + - response_body = { +@pytest.fixture +def single_conversation_response(): + return { "type": "conversation", "id": "151272900024304", "created_at": 1647365706, @@ -213,14 +230,93 @@ def test_conversation_part_has_conversation_id(requests_mock): "total_count": 2, }, } - url = "https://api.intercom.io/conversations/151272900024304" - requests_mock.get(url, json=response_body) - stream1 = ConversationParts(authenticator=NoAuth()) + +@pytest.fixture +def conversation_parts_responses(): + return [ + ( + "https://api.intercom.io/conversations", + build_conversations_response_body( + conversations=[ + {"id":"151272900026677","updated_at":1650988600}, + {"id":"151272900026666","updated_at":1650988500} + ], + next_url="https://api.intercom.io/conversations?per_page=2&page=2" + ) + ), + ( + "https://api.intercom.io/conversations?per_page=2&page=2", + build_conversations_response_body( + conversations=[ + {"id":"151272900026466","updated_at":1650988450}, + {"id":"151272900026680","updated_at":1650988100}, # Older than state, won't be processed + ] + ) + ), + ( + "https://api.intercom.io/conversations/151272900026677", + build_conversation_response_body( + conversation_id="151272900026677", + conversation_parts=[ + {"id": "13740311961","updated_at":1650988300}, + {"id": "13740311962","updated_at":1650988450} + ] + ) + ), + ( + "https://api.intercom.io/conversations/151272900026666", + build_conversation_response_body( + conversation_id="151272900026666", + conversation_parts=[ + {"id": "13740311955","updated_at":1650988150}, + {"id": "13740312056","updated_at":1650988500} + ] + ) + ), + ( + "https://api.intercom.io/conversations/151272900026466", + build_conversation_response_body( + conversation_id="151272900026466", + conversation_parts=[{"id": "13740311970","updated_at":1650988600}] + ) + ) + ] + + +def test_conversation_part_has_conversation_id(requests_mock, single_conversation_response): + """ + Test shows that conversation_part records include the `conversation_id` field. + """ + conversation_id = single_conversation_response["id"] + url = f"https://api.intercom.io/conversations/{conversation_id}" + requests_mock.get(url, json=single_conversation_response) + + conversation_parts = ConversationParts(authenticator=NoAuth()) record_count = 0 - for record in stream1.read_records(sync_mode=SyncMode.incremental, stream_slice={"id": "151272900024304"}): + for record in conversation_parts.read_records(sync_mode=SyncMode.incremental, stream_slice={"id": conversation_id}): assert record["conversation_id"] == "151272900024304" record_count += 1 assert record_count == 2 + + +def test_conversation_part_filtering_based_on_conversation(requests_mock, conversation_parts_responses): + """ + Test shows that conversation_parts filters conversations (from parent stream) correctly + """ + updated_at = 1650988200 + state = {"updated_at": updated_at} + expected_record_ids = set() + for response_tuple in conversation_parts_responses: + requests_mock.register_uri('GET', response_tuple[0], json=response_tuple[1]) + if "conversation_parts" in response_tuple[1]: + expected_record_ids.update([cp["id"] for cp in response_tuple[1]["conversation_parts"]["conversation_parts"]]) + + records = [] + conversation_parts = ConversationParts(authenticator=NoAuth()) + for slice in conversation_parts.stream_slices(sync_mode=SyncMode.incremental, stream_state=state): + records.extend(list(conversation_parts.read_records(sync_mode=SyncMode.incremental, stream_slice=slice, stream_state=state))) + + assert expected_record_ids == {r["id"] for r in records} diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 3ca4791c2981cc..7bc9111ef7b5f7 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -135,7 +135,7 @@ protected List>> discoverInternal(final JdbcData .map(f -> { final Datatype datatype = getFieldType(f); final JsonSchemaType jsonType = getType(datatype); - LOGGER.info("Table {} column {} (type {}[{}]) -> Json type {}", + LOGGER.info("Table {} column {} (type {}[{}]) -> {}", fields.get(0).get(INTERNAL_TABLE_NAME).asText(), f.get(INTERNAL_COLUMN_NAME).asText(), f.get(INTERNAL_COLUMN_TYPE_NAME).asText(), @@ -295,7 +295,7 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { jdbcConfig.get("jdbc_url").asText(), driverClass, streamingQueryConfigProvider, - JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties"), + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties", getJdbcParameterDelimiter()), sourceOperations); quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); @@ -303,4 +303,8 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { return database; } + protected String getJdbcParameterDelimiter() { + return "&"; + } + } diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index 8511ad9d052060..283de93b8d2847 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -25,6 +25,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.models.DbState; @@ -185,6 +186,10 @@ protected String primaryKeyClause(final List columns) { return clause.toString(); } + protected String getJdbcParameterDelimiter() { + return "&"; + } + public void setup() throws Exception { source = getSource(); config = getConfig(); @@ -192,12 +197,14 @@ public void setup() throws Exception { streamName = TABLE_NAME; - database = Databases.createJdbcDatabase( + database = Databases.createStreamingJdbcDatabase( jdbcConfig.get("username").asText(), jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), getDriverClass(), - JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties")); + AdaptiveStreamingQueryConfig::new, + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties", getJdbcParameterDelimiter()), + JdbcUtils.getDefaultSourceOperations()); if (supportsSchemas()) { createSchemas(); diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index 1e2c0c416bc6db..da1a91a9b1730b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index f394f09ac1dadd..d3398f8c4718c6 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -2,7 +2,6 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # - import base64 import json import time @@ -144,9 +143,26 @@ class Cohorts(MixpanelStream): data_field: str = None primary_key: str = "id" + cursor_field = "created" + def path(self, **kwargs) -> str: return "cohorts/list" + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + records = super().parse_response(response, stream_state=stream_state, **kwargs) + for record in records: + record_cursor = record.get(self.cursor_field, "") + state_cursor = stream_state.get(self.cursor_field, "") + if not stream_state or record_cursor >= state_cursor: + yield record + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + state_cursor = (current_stream_state or {}).get(self.cursor_field, "") + + record_cursor = latest_record.get(self.cursor_field, self.start_date) + + return {self.cursor_field: max(state_cursor, record_cursor)} + class FunnelsList(MixpanelStream): """List all funnels @@ -410,6 +426,14 @@ class Engage(MixpanelStream): page_size: int = 1000 # min 100 _total: Any = None + @property + def source_defined_cursor(self) -> bool: + return False + + @property + def supports_incremental(self) -> bool: + return True + # enable automatic object mutation to align with desired schema before outputting to the destination transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -448,7 +472,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, self._total = None return None - def process_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + def process_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: """ { "page": 0 @@ -472,6 +496,7 @@ def process_response(self, response: requests.Response, **kwargs) -> Iterable[Ma "$name":"Nadine Burzler" "id":"632540fa-d1af-4535-bc52-e331955d363e" "$last_seen":"2020-06-28T12:12:31" + ... } },{ ... @@ -481,6 +506,7 @@ def process_response(self, response: requests.Response, **kwargs) -> Iterable[Ma } """ records = response.json().get(self.data_field, {}) + cursor_field = stream_state.get(self.usr_cursor_key()) for record in records: item = {"distinct_id": record["$distinct_id"]} properties = record["$properties"] @@ -492,7 +518,10 @@ def process_response(self, response: requests.Response, **kwargs) -> Iterable[Ma # to stream: 'browser' this_property_name = this_property_name[1:] item[this_property_name] = properties[property_name] - yield item + item_cursor = item.get(cursor_field, "") + state_cursor = stream_state.get(cursor_field, "") + if not stream_state or item_cursor >= state_cursor: + yield item def get_json_schema(self) -> Mapping[str, Any]: """ @@ -533,6 +562,32 @@ def get_json_schema(self) -> Mapping[str, Any]: return schema + def usr_cursor_key(self): + return "usr_cursor_key" + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + if sync_mode == SyncMode.incremental: + cursor_name = cursor_field[-1] + if stream_state: + stream_state[self.usr_cursor_key()] = cursor_name + else: + stream_state = {self.usr_cursor_key(): cursor_name} + return super().read_records(sync_mode, cursor_field, stream_slice, stream_state=stream_state) + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + cursor_field = current_stream_state.get(self.usr_cursor_key()) + state_cursor = (current_stream_state or {}).get(cursor_field, "") + + record_cursor = latest_record.get(cursor_field, self.start_date) + + return {cursor_field: max(state_cursor, record_cursor)} + class CohortMembers(Engage): """Return list of users grouped by cohort""" @@ -550,7 +605,9 @@ def stream_slices( self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: stream_slices = [] - cohorts = Cohorts(**self.get_stream_params()).read_records(sync_mode=sync_mode) + # full refresh is needed because even though some cohorts might already have been read + # they can still have new members added + cohorts = Cohorts(**self.get_stream_params()).read_records(SyncMode.full_refresh) for cohort in cohorts: stream_slices.append({"id": cohort["id"]}) @@ -788,12 +845,6 @@ def get_json_schema(self) -> Mapping[str, Any]: return schema -class TokenAuthenticatorBase64(TokenAuthenticator): - def __init__(self, token: str, auth_method: str = "Basic", **kwargs): - token = base64.b64encode(token.encode("utf8")).decode("utf8") - super().__init__(token=token, auth_method=auth_method, **kwargs) - - class SourceMixpanel(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: """ @@ -856,3 +907,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Funnels(authenticator=auth, **config), Revenue(authenticator=auth, **config), ] + + +class TokenAuthenticatorBase64(TokenAuthenticator): + def __init__(self, token: str, auth_method: str = "Basic", **kwargs): + token = base64.b64encode(token.encode("utf8")).decode("utf8") + super().__init__(token=token, auth_method=auth_method, **kwargs) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index aa2623f83ea9ae..279042369777e3 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -9,6 +9,9 @@ from airbyte_cdk.models import SyncMode from source_mixpanel.source import ( Annotations, + CohortMembers, + Cohorts, + Engage, EngageSchema, Export, ExportSchema, @@ -23,6 +26,8 @@ logger = AirbyteLogger() +MIXPANEL_BASE_URL = "https://mixpanel.com/api/2.0/" + @pytest.fixture def patch_base_class(mocker): @@ -69,9 +74,107 @@ def test_updated_state(patch_incremental_base_class): assert updated_state == {"date": "2021-02-25T00:00:00Z"} -def test_cohorts_stream(): - # tested in itaseskii:mixpanel-incremental-syncs - return None +@pytest.fixture +def cohorts_response(): + return setup_response( + 200, + [ + { + "count": 150, + "is_visible": 1, + "description": "This cohort is visible, has an id = 1000, and currently has 150 users.", + "created": "2019-03-19 23:49:51", + "project_id": 1, + "id": 1000, + "name": "Cohort One", + }, + { + "count": 25, + "is_visible": 0, + "description": "This cohort isn't visible, has an id = 2000, and currently has 25 users.", + "created": "2019-04-02 23:22:01", + "project_id": 1, + "id": 2000, + "name": "Cohort Two", + }, + ], + ) + + +def test_cohorts_stream_incremental(requests_mock, cohorts_response): + requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", cohorts_response) + + stream = Cohorts(authenticator=MagicMock()) + + records = stream.read_records(sync_mode=SyncMode.incremental, stream_state={"created": "2019-04-02 23:22:01"}) + + records_length = sum(1 for _ in records) + assert records_length == 1 + + +@pytest.fixture +def engage_response(): + return setup_response( + 200, + { + "page": 0, + "page_size": 1000, + "session_id": "1234567890-EXAMPL", + "status": "ok", + "total": 2, + "results": [ + { + "$distinct_id": "9d35cd7f-3f06-4549-91bf-198ee58bb58a", + "$properties": { + "$created": "2008-12-12T11:20:47", + "$browser": "Chrome", + "$browser_version": "83.0.4103.116", + "$email": "clark@asw.com", + "$first_name": "Clark", + "$last_name": "Kent", + "$name": "Clark Kent", + }, + }, + { + "$distinct_id": "cd9d357f-3f06-4549-91bf-158bb598ee8a", + "$properties": { + "$created": "2008-11-12T11:20:47", + "$browser": "Firefox", + "$browser_version": "83.0.4103.116", + "$email": "bruce@asw.com", + "$first_name": "Bruce", + "$last_name": "Wayne", + "$name": "Bruce Wayne", + }, + }, + ], + }, + ) + + +def test_engage_stream_incremental(requests_mock, engage_response): + requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000", engage_response) + + stream = Engage(authenticator=MagicMock()) + + records = stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["created"], stream_state={"created": "2008-12-12T11:20:47"}) + + records_length = sum(1 for _ in records) + assert records_length == 1 + + +def test_cohort_members_stream_incremental(requests_mock, engage_response, cohorts_response): + requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000", engage_response) + requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", cohorts_response) + + stream = CohortMembers(authenticator=MagicMock()) + + records = stream.read_records( + sync_mode=SyncMode.incremental, cursor_field=["created"], stream_state={"created": "2008-12-12T11:20:47"}, stream_slice={"id": 1000} + ) + + records_length = sum(1 for _ in records) + assert records_length == 1 @pytest.fixture diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 015453ecde72ad..ab3e22af5dcbce 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.3.22 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index f826af5a2e5287..8091bbc9be8f09 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.21 +LABEL io.airbyte.version=0.3.22 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index a7759c8c9a5d88..4a95c3c9cd047e 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.5.10 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index 6484aed0460a17..d6d38b2419c83c 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -36,7 +36,6 @@ class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTes protected static final String TEST_PASSWORD = "test"; protected static MySQLContainer container; - protected JsonNode config; protected Database database; @BeforeAll @@ -47,7 +46,7 @@ static void init() throws SQLException { .withEnv("MYSQL_ROOT_HOST", "%") .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD); container.start(); - final Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", TEST_PASSWORD); + final Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", container.getPassword()); connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); } diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 952f96f2c0d7c6..1072e6d436724d 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.5.9 +LABEL io.airbyte.version=0.5.10 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile index f64d7feb84a271..96726701554ad1 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile @@ -17,5 +17,5 @@ ENV TZ UTC COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.3.15 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java index 0705682671001f..f012936d4649e9 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java @@ -16,15 +16,14 @@ import io.airbyte.db.jdbc.JdbcUtils; import java.sql.SQLException; import java.util.List; -import java.util.stream.Collectors; import org.junit.jupiter.api.Test; public class OracleSourceNneAcceptanceTest extends OracleStrictEncryptSourceAcceptanceTest { @Test - public void testEncrytion() throws SQLException { - final JsonNode clone = Jsons.clone(getConfig()); - ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() + public void testEncryption() throws SQLException { + final ObjectNode clone = (ObjectNode) Jsons.clone(getConfig()); + clone.set("encryption", Jsons.jsonNode(ImmutableMap.builder() .put("encryption_method", "client_nne") .put("encryption_algorithm", "3DES168") .build())); @@ -45,7 +44,7 @@ public void testEncrytion() throws SQLException { final String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; - final List collect = database.unsafeQuery(network_service_banner).collect(Collectors.toList()); + final List collect = database.unsafeQuery(network_service_banner).toList(); assertTrue(collect.get(2).get("NETWORK_SERVICE_BANNER").asText() .contains(algorithm + " Encryption")); @@ -53,8 +52,8 @@ public void testEncrytion() throws SQLException { @Test public void testCheckProtocol() throws SQLException { - final JsonNode clone = Jsons.clone(getConfig()); - ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() + final ObjectNode clone = (ObjectNode) Jsons.clone(getConfig()); + clone.set("encryption", Jsons.jsonNode(ImmutableMap.builder() .put("encryption_method", "client_nne") .put("encryption_algorithm", "AES256") .build())); @@ -70,11 +69,10 @@ public void testCheckProtocol() throws SQLException { clone.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm + " )")); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); final String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; - final List collect = database.unsafeQuery(network_service_banner).collect(Collectors.toList()); + final List collect = database.unsafeQuery(network_service_banner).toList(); assertEquals("tcp", collect.get(0).get("NETWORK_PROTOCOL").asText()); } diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java index 150c47cc904589..0106bd75a51292 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java @@ -61,8 +61,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get("port").asText(), config.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + - "oracle.net.encryption_types_client=( 3DES168 )")); + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( 3DES168 )", ";")); database.execute(connection -> { connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java index 437869f2de4639..6311c64b820a04 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java @@ -56,14 +56,6 @@ class OracleStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTe @BeforeAll static void init() { - ORACLE_DB = new OracleContainer("epiclabs/docker-oracle-xe-11g") - .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD"); - ORACLE_DB.start(); - } - - @BeforeEach - public void setup() throws Exception { - SCHEMA_NAME = "JDBC_INTEGRATION_TEST1"; SCHEMA_NAME2 = "JDBC_INTEGRATION_TEST2"; TEST_SCHEMAS = ImmutableSet.of(SCHEMA_NAME, SCHEMA_NAME2); @@ -84,6 +76,13 @@ public void setup() throws Exception { ID_VALUE_4 = new BigDecimal(4); ID_VALUE_5 = new BigDecimal(5); + ORACLE_DB = new OracleContainer("epiclabs/docker-oracle-xe-11g") + .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD"); + ORACLE_DB.start(); + } + + @BeforeEach + public void setup() throws Exception { config = Jsons.jsonNode(ImmutableMap.builder() .put("host", ORACLE_DB.getHost()) .put("port", ORACLE_DB.getFirstMappedPort()) @@ -126,9 +125,8 @@ void cleanUpTables() throws SQLException { conn.createStatement().executeQuery(String.format("SELECT TABLE_NAME FROM ALL_TABLES WHERE OWNER = '%s'", schemaName)); while (resultSet.next()) { final String tableName = resultSet.getString("TABLE_NAME"); - final String tableNameProcessed = tableName.contains(" ") ? sourceOperations - .enquoteIdentifier(conn, tableName) : tableName; - conn.createStatement().executeQuery(String.format("DROP TABLE %s.%s", schemaName, tableNameProcessed)); + final String tableNameProcessed = tableName.contains(" ") ? sourceOperations.enquoteIdentifier(conn, tableName) : tableName; + conn.createStatement().executeQuery("DROP TABLE " + schemaName + "." + tableNameProcessed); } } if (!conn.isClosed()) @@ -180,7 +178,12 @@ public void createSchemas() throws SQLException { } } - public void executeOracleStatement(final String query) throws SQLException { + @Override + protected String getJdbcParameterDelimiter() { + return ";"; + } + + public void executeOracleStatement(final String query) { try (final Connection conn = DriverManager.getConnection( ORACLE_DB.getJdbcUrl(), ORACLE_DB.getUsername(), @@ -194,8 +197,8 @@ public void executeOracleStatement(final String query) throws SQLException { public static void logSQLException(final SQLException ex) { for (final Throwable e : ex) { - if (e instanceof SQLException) { - if (ignoreSQLException(((SQLException) e).getSQLState()) == false) { + if (e instanceof final SQLException sqlException) { + if (!ignoreSQLException(sqlException.getSQLState())) { LOGGER.info("SQLState: " + ((SQLException) e).getSQLState()); LOGGER.info("Error Code: " + ((SQLException) e).getErrorCode()); LOGGER.info("Message: " + e.getMessage()); diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index dc2ba25e64557e..cfc0c7aeaa3982 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -8,5 +8,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.14 +LABEL io.airbyte.version=0.3.15 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index 94813f54fff3e8..acd8e04c6a58ba 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -18,6 +18,7 @@ import io.airbyte.protocol.models.CommonField; import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.sql.JDBCType; import java.util.ArrayList; import java.util.List; @@ -92,7 +93,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { } } if (!additionalParameters.isEmpty()) { - final String connectionParams = String.join(";", additionalParameters); + final String connectionParams = String.join(getJdbcParameterDelimiter(), additionalParameters); configBuilder.put("connection_properties", connectionParams); } @@ -129,7 +130,7 @@ private Protocol obtainConnectionProtocol(final JsonNode encryption, final List< private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); - try (final PrintWriter out = new PrintWriter("certificate.pem")) { + try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) { out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); @@ -170,6 +171,11 @@ public Set getExcludedInternalNameSpaces() { return Set.of(); } + @Override + protected String getJdbcParameterDelimiter() { + return ";"; + } + public static void main(final String[] args) throws Exception { final Source source = OracleSource.sshWrappedSource(); LOGGER.info("starting source: {}", OracleSource.class); diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java index 0d5c7a604403f0..a608165340c964 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java @@ -55,15 +55,6 @@ class OracleJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { @BeforeAll static void init() { // Oracle returns uppercase values - - ORACLE_DB = new OracleContainer("epiclabs/docker-oracle-xe-11g") - .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD"); - ORACLE_DB.start(); - } - - @BeforeEach - public void setup() throws Exception { - SCHEMA_NAME = "JDBC_INTEGRATION_TEST1"; SCHEMA_NAME2 = "JDBC_INTEGRATION_TEST2"; TEST_SCHEMAS = ImmutableSet.of(SCHEMA_NAME, SCHEMA_NAME2); @@ -84,6 +75,13 @@ public void setup() throws Exception { ID_VALUE_4 = new BigDecimal(4); ID_VALUE_5 = new BigDecimal(5); + ORACLE_DB = new OracleContainer("epiclabs/docker-oracle-xe-11g") + .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD"); + ORACLE_DB.start(); + } + + @BeforeEach + public void setup() throws Exception { config = Jsons.jsonNode(ImmutableMap.builder() .put("host", ORACLE_DB.getHost()) .put("port", ORACLE_DB.getFirstMappedPort()) @@ -124,7 +122,7 @@ void cleanUpTables() throws SQLException { final String tableName = resultSet.getString("TABLE_NAME"); final String tableNameProcessed = tableName.contains(" ") ? sourceOperations .enquoteIdentifier(conn, tableName) : tableName; - conn.createStatement().executeQuery(String.format("DROP TABLE %s.%s", schemaName, tableNameProcessed)); + conn.createStatement().executeQuery("DROP TABLE " + schemaName + "." + tableNameProcessed); } } if (!conn.isClosed()) diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java index 529cbaeff4c924..ff86b31c011fdc 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java @@ -24,6 +24,7 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,8 +46,8 @@ class OracleSourceTest { Field.of("IMAGE", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))); private static final ConfiguredAirbyteCatalog CONFIGURED_CATALOG = CatalogHelpers.toDefaultConfiguredCatalog(CATALOG); - private static final Set ASCII_MESSAGES = Sets.newHashSet( - createRecord(STREAM_NAME, map("ID", new BigDecimal("1.0"), "NAME", "user", "IMAGE", "last_summer.png".getBytes()))); + private static final Set ASCII_MESSAGES = Sets.newHashSet(createRecord(STREAM_NAME, + map("ID", new BigDecimal("1.0"), "NAME", "user", "IMAGE", "last_summer.png".getBytes(StandardCharsets.UTF_8)))); private static OracleContainer ORACLE_DB; diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java index cdca797efd006d..cbc25fd3aa208e 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java @@ -38,18 +38,18 @@ class OracleStressTest extends JdbcStressTest { @BeforeAll static void init() { - ORACLE_DB = new OracleContainer("epiclabs/docker-oracle-xe-11g"); - ORACLE_DB.start(); - } - - @BeforeEach - public void setup() throws Exception { TABLE_NAME = "ID_AND_NAME"; COL_ID = "ID"; COL_NAME = "NAME"; COL_ID_TYPE = "NUMBER(38,0)"; INSERT_STATEMENT = "INTO id_and_name (id, name) VALUES (%s,'picard-%s')"; + ORACLE_DB = new OracleContainer("epiclabs/docker-oracle-xe-11g"); + ORACLE_DB.start(); + } + + @BeforeEach + public void setup() throws Exception { config = Jsons.jsonNode(ImmutableMap.builder() .put("host", ORACLE_DB.getHost()) .put("port", ORACLE_DB.getFirstMappedPort()) @@ -57,7 +57,6 @@ public void setup() throws Exception { .put("username", ORACLE_DB.getUsername()) .put("password", ORACLE_DB.getPassword()) .build()); - super.setup(); } diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 6b6b28beb9abb3..e2fdc14b44d95a 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.4.12 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 6e5cc0d203778b..9ae29ae6eb2250 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.11 +LABEL io.airbyte.version=0.4.12 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/config.json b/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/config.json index 2e4bbdfb3b61dd..4a9d0b46c3bf09 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/config.json +++ b/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/config.json @@ -1 +1 @@ -{ "start_date": "2021-04-01", "base": "USD" } +{ "start_date": "2021-04-01", "base": "USD", "access_key": "abcdef" } diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/configured_catalog.json index 8c34f50528be39..c42547264dd573 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/configured_catalog.json @@ -7,6 +7,9 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { + "access_key": { + "type": "string" + }, "base": { "type": "string" }, diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/invalid_config.json index 779b9ee5d1e616..9daf8e2f3fe7ec 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/invalid_config.json +++ b/airbyte-integrations/connectors/source-python-http-tutorial/sample_files/invalid_config.json @@ -1 +1 @@ -{ "start_date": "2021-04-01", "base": "BTC" } +{ "start_date": "2021-04-01", "base": "BTC", "access_key": "abcdef" } diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/schemas/exchange_rates.json b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/schemas/exchange_rates.json index 80b47d0eeeee44..84b6325ce5d24b 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/schemas/exchange_rates.json +++ b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/schemas/exchange_rates.json @@ -2,6 +2,9 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { + "access_key": { + "type": "string" + }, "base": { "type": "string" }, diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py index 2f8c75c5e74c98..76b6af1f6eced6 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py @@ -14,14 +14,16 @@ class ExchangeRates(HttpStream): - url_base = "https://api.exchangeratesapi.io/" + url_base = "http://api.exchangeratesapi.io/" cursor_field = "date" primary_key = "date" - def __init__(self, base: str, start_date: datetime, **kwargs): - super().__init__(**kwargs) - self.base = base + def __init__(self, config: Mapping[str, Any], start_date: datetime, **kwargs): + super().__init__() + self.base = config["base"] + self.access_key = config["access_key"] self.start_date = start_date + self._cursor_value = None def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: # The API does not offer pagination, so we return None to indicate there are no more pages in the response @@ -38,8 +40,8 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - # The api requires that we include the base currency as a query param so we do that in this method - return {"base": self.base} + # The api requires that we include access_key as a query param so we do that in this method + return {"access_key": self.access_key} def parse_response( self, @@ -104,4 +106,4 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = NoAuth() # Parse the date from a string into a datetime object start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") - return [ExchangeRates(authenticator=auth, base=config["base"], start_date=start_date)] + return [ExchangeRates(authenticator=auth, config=config, start_date=start_date)] diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/spec.json b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/spec.json index c62b4b93f01428..94f00e9b0e1682 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/spec.json +++ b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/spec.json @@ -7,6 +7,11 @@ "required": ["start_date", "base"], "additionalProperties": false, "properties": { + "access_key": { + "title": "Access Key", + "type": "string", + "description": "API access key used to retrieve data from the Exchange Rates API." + }, "start_date": { "title": "Start Date", "type": "string", diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile index af8cc61f418734..a743455d2227c1 100644 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ b/airbyte-integrations/connectors/source-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.9 +LABEL io.airbyte.version=0.3.10 LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java index 1f8e82d00496f7..e2be871f2000e7 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.NoOpStreamingQueryConfig; +import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; @@ -23,9 +23,9 @@ public class ScaffoldJavaJdbcSource extends AbstractJdbcSource impleme static final String DRIVER_CLASS = "driver_name_here"; public ScaffoldJavaJdbcSource() { - // By default, NoOpStreamingQueryConfig class is used. If the JDBC supports custom - // fetch size, change it to AdaptiveStreamingQueryConfig for better performance. - super(DRIVER_CLASS, NoOpStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); + // TODO: if the JDBC driver does not support custom fetch size, use NoOpStreamingQueryConfig + // instead of AdaptiveStreamingQueryConfig. + super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); } // TODO The config is based on spec.json, update according to your DB diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml index a625390b4d5eb6..97cf9c7d8e1cd9 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml @@ -15,12 +15,12 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes incremental: # TODO if your connector does not implement incremental sync, remove this block - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index f2d1461977b569..7d904bc10687fd 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile index 6cd5e75bcfbe12..6179f1f2b654b9 100755 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ b/airbyte-integrations/connectors/source-tidb/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-tidb COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-tidb diff --git a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java index 370236b8d69172..f10efc8b6e163f 100644 --- a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java +++ b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java @@ -8,7 +8,7 @@ import com.google.common.collect.ImmutableMap; import com.mysql.cj.MysqlType; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.streaming.NoOpStreamingQueryConfig; +import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshWrappedSource; @@ -33,7 +33,7 @@ public static Source sshWrappedSource() { } public TiDBSource() { - super(DRIVER_CLASS, NoOpStreamingQueryConfig::new, new TiDBSourceOperations()); + super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new TiDBSourceOperations()); } @Override diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index 5524acd9717b7c..d9fcada3aa5d46 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index da6cad26a536c7..e008d0a54887e5 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -10,8 +10,7 @@ "default": {}, "order": 0, "type": "object", - "oneOf": [ - { + "oneOf": [{ "title": "OAuth2.0", "type": "object", "properties": { @@ -35,14 +34,13 @@ }, "access_token": { "title": "Access Token", - "description": "Long-term Authorized Access Token.", + "description": "The long-term authorized access token.", "airbyte_secret": true, "type": "string" } }, "required": ["app_id", "secret", "access_token"] - }, - { + }, { "title": "Production Access Token", "type": "object", "properties": { @@ -65,14 +63,13 @@ }, "access_token": { "title": "Access Token", - "description": "The Long-term Authorized Access Token.", + "description": "The long-term authorized access token.", "airbyte_secret": true, "type": "string" } }, "required": ["app_id", "secret", "access_token"] - }, - { + }, { "title": "Sandbox Access Token", "type": "object", "properties": { @@ -84,12 +81,12 @@ }, "advertiser_id": { "title": "Advertiser ID", - "description": "The Advertiser ID which generated for the developer's Sandbox application.", + "description": "The Advertiser ID which generated for the developer's Sandbox application.", "type": "string" }, "access_token": { "title": "Access Token", - "description": "The Long-term Authorized Access Token.", + "description": "The long-term authorized access token.", "airbyte_secret": true, "type": "string" } @@ -100,7 +97,7 @@ }, "start_date": { "title": "Start Date *", - "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", + "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated.If this parameter is not set, all data will be replicated.", "default": "2016-09-01", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "order": 1, @@ -108,7 +105,7 @@ }, "report_granularity": { "title": "Report Granularity *", - "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.", + "description": "Grouping of your reports based on time. Lifetime will have no grouping. This option is used for reports' streams only.", "default": "DAY", "enum": ["LIFETIME", "DAY", "HOUR"], "order": 2, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py index f33e829befff06..6ce93d4b24e275 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py @@ -23,7 +23,7 @@ class Config: secret: str = Field(title="Secret", description="The private key of the developer's application.", airbyte_secret=True) - access_token: str = Field(title="Access Token", description="Long-term Authorized Access Token.", airbyte_secret=True) + access_token: str = Field(title="Access Token", description="The long-term authorized access token.", airbyte_secret=True) class SandboxEnvSpec(BaseModel): @@ -34,10 +34,10 @@ class Config: # it is string because UI has the bug https://github.com/airbytehq/airbyte/issues/6875 advertiser_id: str = Field( - title="Advertiser ID", description="The Advertiser ID which generated for the developer's Sandbox application." + title="Advertiser ID", description="The Advertiser ID which generated for the developer's Sandbox application." ) - access_token: str = Field(title="Access Token", description="The Long-term Authorized Access Token.", airbyte_secret=True) + access_token: str = Field(title="Access Token", description="The long-term authorized access token.", airbyte_secret=True) class ProductionEnvSpec(BaseModel): @@ -50,7 +50,7 @@ class Config: app_id: str = Field(description="The App ID applied by the developer.", title="App ID") secret: str = Field(title="Secret", description="The private key of the developer application.", airbyte_secret=True) - access_token: str = Field(title="Access Token", description="The Long-term Authorized Access Token.", airbyte_secret=True) + access_token: str = Field(title="Access Token", description="The long-term authorized access token.", airbyte_secret=True) class SourceTiktokMarketingSpec(BaseModel): @@ -65,15 +65,14 @@ class Config: title="Start Date *", default=DEFAULT_START_DATE, pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - description="The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. " + description="The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated." "If this parameter is not set, all data will be replicated.", order=1, ) report_granularity: str = Field( title="Report Granularity *", - description="Which time granularity should be grouped by; for LIFETIME there will be no grouping. " - "This option is used for reports' streams only.", + description="Grouping of your reports based on time. Lifetime will have no grouping. This option is used for reports' streams only.", default=ReportGranularity.default().value, enum=[g.value for g in ReportGranularity], order=2, diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index 2282a6923040b2..25890b1690b434 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -1,7 +1,7 @@ ARG JDK_VERSION=17.0.1 FROM openjdk:${JDK_VERSION}-slim AS metrics-reporter -ARG VERSION=0.36.5-alpha +ARG VERSION=0.36.6-alpha ENV APPLICATION airbyte-metrics-reporter ENV VERSION ${VERSION} diff --git a/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java index f9d2bf5b0492d0..9d063088cf5a9a 100644 --- a/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java +++ b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java @@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import java.util.Objects; public class JsonSchemaType { @@ -51,17 +52,17 @@ private Builder(final JsonSchemaPrimitive type) { typeMapBuilder.put(TYPE, type.name().toLowerCase()); } - public Builder withFormat(String value) { + public Builder withFormat(final String value) { typeMapBuilder.put(FORMAT, value); return this; } - public Builder withContentEncoding(String value) { + public Builder withContentEncoding(final String value) { typeMapBuilder.put(CONTENT_ENCODING, value); return this; } - public Builder withAirbyteType(String value) { + public Builder withAirbyteType(final String value) { typeMapBuilder.put(AIRBYTE_TYPE, value); return this; } @@ -72,4 +73,25 @@ public JsonSchemaType build() { } + @Override + public String toString() { + return String.format("JsonSchemaType(%s)", jsonSchemaTypeMap.toString()); + } + + @Override + public boolean equals(final Object other) { + if (other == null) { + return false; + } + if (!(other instanceof final JsonSchemaType that)) { + return false; + } + return Objects.equals(this.jsonSchemaTypeMap, that.jsonSchemaTypeMap); + } + + @Override + public int hashCode() { + return Objects.hashCode(this.jsonSchemaTypeMap); + } + } diff --git a/airbyte-scheduler/app/Dockerfile b/airbyte-scheduler/app/Dockerfile index b708b1982cf318..b841e267f1ed10 100644 --- a/airbyte-scheduler/app/Dockerfile +++ b/airbyte-scheduler/app/Dockerfile @@ -1,7 +1,7 @@ ARG JDK_VERSION=17.0.1 FROM openjdk:${JDK_VERSION}-slim AS scheduler -ARG VERSION=0.36.5-alpha +ARG VERSION=0.36.6-alpha ENV APPLICATION airbyte-scheduler ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index a6c2b94c27ca76..6f514552ec5aa4 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,7 +3,7 @@ FROM openjdk:${JDK_VERSION}-slim AS server EXPOSE 8000 -ARG VERSION=0.36.5-alpha +ARG VERSION=0.36.6-alpha ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 0aa1fa28a709bf..3625e37351bde8 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "airbyte-webapp", - "version": "0.36.5-alpha", + "version": "0.36.6-alpha", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "airbyte-webapp", - "version": "0.36.5-alpha", + "version": "0.36.6-alpha", "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.1.1", "@fortawesome/free-brands-svg-icons": "^6.1.1", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 1fbc245d3d6ffa..317d3635b31b4a 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.36.5-alpha", + "version": "0.36.6-alpha", "private": true, "engines": { "node": ">=16.0.0" @@ -9,6 +9,7 @@ "start": "react-scripts start", "build": "react-scripts build", "test": "react-scripts test", + "test:coverage": "npm test -- --coverage --watchAll=false", "format": "prettier --write 'src/**/*.{ts,tsx}'", "storybook": "start-storybook -p 9009 -s public --quiet", "lint": "eslint --ext js,ts,tsx src", diff --git a/airbyte-webapp/src/components/StatusIcon/CircleLoader.tsx b/airbyte-webapp/src/components/StatusIcon/CircleLoader.tsx index 20cc81a4fa794e..49de2a314da7a4 100644 --- a/airbyte-webapp/src/components/StatusIcon/CircleLoader.tsx +++ b/airbyte-webapp/src/components/StatusIcon/CircleLoader.tsx @@ -38,8 +38,8 @@ const CircleLoader = ({ title }: Props): JSX.Element => ( gradientUnits="userSpaceOnUse" gradientTransform="translate(0 0)" > - - + + {title && {title}} @@ -47,15 +47,15 @@ const CircleLoader = ({ title }: Props): JSX.Element => ( diff --git a/airbyte-webapp/src/components/base/Input/Input.test.tsx b/airbyte-webapp/src/components/base/Input/Input.test.tsx new file mode 100644 index 00000000000000..d411053912237a --- /dev/null +++ b/airbyte-webapp/src/components/base/Input/Input.test.tsx @@ -0,0 +1,44 @@ +import { render } from "utils/testutils"; + +import { Input } from "./Input"; + +describe("", () => { + test("renders text input", async () => { + const value = "aribyte@example.com"; + const { getByTestId, queryByTestId } = await render(); + + expect(getByTestId("input")).toHaveAttribute("type", "text"); + expect(getByTestId("input")).toHaveValue(value); + expect(queryByTestId("toggle-password-visibility-button")).toBeFalsy(); + }); + + test("renders another type of input", async () => { + const type = "number"; + const value = 888; + const { getByTestId, queryByTestId } = await render(); + + expect(getByTestId("input")).toHaveAttribute("type", type); + expect(getByTestId("input")).toHaveValue(value); + expect(queryByTestId("toggle-password-visibility-button")).toBeFalsy(); + }); + + test("renders password input with visibilty button", async () => { + const value = "eight888"; + const { getByTestId, getByRole } = await render(); + + expect(getByTestId("input")).toHaveAttribute("type", "password"); + expect(getByTestId("input")).toHaveValue(value); + expect(getByRole("img", { hidden: true })).toHaveAttribute("data-icon", "eye"); + }); + + test("renders visible password when visibility button is clicked", async () => { + const value = "eight888"; + const { getByTestId, getByRole } = await render(); + + getByTestId("toggle-password-visibility-button")?.click(); + + expect(getByTestId("input")).toHaveAttribute("type", "text"); + expect(getByTestId("input")).toHaveValue(value); + expect(getByRole("img", { hidden: true })).toHaveAttribute("data-icon", "eye-slash"); + }); +}); diff --git a/airbyte-webapp/src/components/base/Input/Input.tsx b/airbyte-webapp/src/components/base/Input/Input.tsx index 796d12fa880cc9..117ff76e206a51 100644 --- a/airbyte-webapp/src/components/base/Input/Input.tsx +++ b/airbyte-webapp/src/components/base/Input/Input.tsx @@ -1,6 +1,8 @@ import { faEye, faEyeSlash } from "@fortawesome/free-regular-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React, { useState } from "react"; +import React from "react"; +import { useIntl } from "react-intl"; +import { useToggle } from "react-use"; import styled from "styled-components"; import { Theme } from "theme"; @@ -18,37 +20,44 @@ const getBackgroundColor = (props: IStyleProps) => { return props.theme.greyColor0; }; -export type InputProps = { +export interface InputProps extends React.InputHTMLAttributes { error?: boolean; light?: boolean; -} & React.InputHTMLAttributes; +} -const InputComponent = styled.input` - outline: none; +const InputContainer = styled.div` width: 100%; - padding: 7px 18px 7px 8px; - border-radius: 4px; - font-size: 14px; - line-height: 20px; - font-weight: normal; - border: 1px solid ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor0)}; + position: relative; background: ${(props) => getBackgroundColor(props)}; - color: ${({ theme }) => theme.textColor}; - caret-color: ${({ theme }) => theme.primaryColor}; - - &::placeholder { - color: ${({ theme }) => theme.greyColor40}; - } + border: 1px solid ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor0)}; + border-radius: 4px; &:hover { background: ${({ theme, light }) => (light ? theme.whiteColor : theme.greyColor20)}; border-color: ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor20)}; } - &:focus { + &.input-container--focused { background: ${({ theme, light }) => (light ? theme.whiteColor : theme.primaryColor12)}; border-color: ${({ theme }) => theme.primaryColor}; } +`; + +const InputComponent = styled.input` + outline: none; + width: ${({ isPassword }) => (isPassword ? "calc(100% - 22px)" : "100%")}; + padding: 7px 8px 7px 8px; + font-size: 14px; + line-height: 20px; + font-weight: normal; + border: none; + background: none; + color: ${({ theme }) => theme.textColor}; + caret-color: ${({ theme }) => theme.primaryColor}; + + &::placeholder { + color: ${({ theme }) => theme.greyColor40}; + } &:disabled { pointer-events: none; @@ -56,34 +65,53 @@ const InputComponent = styled.input` } `; -const Container = styled.div` - width: 100%; - position: relative; -`; - const VisibilityButton = styled(Button)` position: absolute; - right: 2px; - top: 7px; + right: 0px; + top: 0; + display: flex; + height: 100%; + width: 30px; + align-items: center; + justify-content: center; + border: none; `; const Input: React.FC = (props) => { - const [isContentVisible, setIsContentVisible] = useState(false); - - if (props.type === "password") { - return ( - - - {props.disabled ? null : ( - setIsContentVisible(!isContentVisible)} type="button"> - - - )} - - ); - } - - return ; + const { formatMessage } = useIntl(); + const [isContentVisible, setIsContentVisible] = useToggle(false); + const [focused, toggleFocused] = useToggle(false); + + const isPassword = props.type === "password"; + const isVisibilityButtonVisible = isPassword && !props.disabled; + const type = isPassword ? (isContentVisible ? "text" : "password") : props.type; + const onInputFocusChange = () => toggleFocused(); + + return ( + + + {isVisibilityButtonVisible ? ( + setIsContentVisible()} + type="button" + aria-label={formatMessage({ + id: `ui.input.${isContentVisible ? "hide" : "show"}Password`, + })} + data-testid="toggle-password-visibility-button" + > + + + ) : null} + + ); }; export default Input; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 7e6de2b13dae93..76c5831e364b51 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -490,6 +490,8 @@ "errorView.notFound": "Resource not found", "errorView.unknown": "Unknown", + "ui.input.showPassword": "Show password", + "ui.input.hidePassword": "Hide password", "ui.keyValuePair": "{key}: {value}", "ui.keyValuePairV2": "{key} ({value})", "ui.keyValuePairV3": "{key}, {value}", diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json index 245da9e332a10d..6055137c05aae9 100644 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ b/airbyte-webapp/src/packages/cloud/locales/en.json @@ -7,6 +7,8 @@ "login.resendEmail": "Didn’t receive the email? Send it again", "login.yourEmail": "Your work email*", "login.yourEmail.placeholder": "christophersmith@gmail.com", + "login.yourEmail.notFound": "User not found", + "login.unknownError": "An unknown error has occurred", "login.password": "Enter your password*", "login.password.placeholder": "Enter a strong password", "login.yourPassword": "Enter your password*", diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx index e59bce31abddb5..382364266bd10c 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx @@ -19,7 +19,7 @@ const ResetPasswordPageValidationSchema = yup.object().shape({ const ResetPasswordPage: React.FC = () => { const { requirePasswordReset } = useAuthService(); const { registerNotification } = useNotificationService(); - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); return (
@@ -32,13 +32,19 @@ const ResetPasswordPage: React.FC = () => { email: "", }} validationSchema={ResetPasswordPageValidationSchema} - onSubmit={async ({ email }) => { - await requirePasswordReset(email); - registerNotification({ - id: "resetPassword.emailSent", - title: formatMessage({ id: "login.resetPassword.emailSent" }), - isError: false, - }); + onSubmit={async ({ email }, FormikBag) => { + try { + await requirePasswordReset(email); + registerNotification({ + id: "resetPassword.emailSent", + title: formatMessage({ id: "login.resetPassword.emailSent" }), + isError: false, + }); + } catch (err) { + err.message.includes("user-not-found") + ? FormikBag.setFieldError("email", "login.yourEmail.notFound") + : FormikBag.setFieldError("email", "login.unknownError"); + } }} validateOnBlur={true} validateOnChange={false} diff --git a/airbyte-webapp/src/utils/testutils.tsx b/airbyte-webapp/src/utils/testutils.tsx index ab6a2f354d14c3..9f9e50e70fc903 100644 --- a/airbyte-webapp/src/utils/testutils.tsx +++ b/airbyte-webapp/src/utils/testutils.tsx @@ -1,5 +1,4 @@ -import { act, Queries, render as rtlRender, RenderResult } from "@testing-library/react"; -import { History } from "history"; +import { act, Queries, queries, render as rtlRender, RenderOptions, RenderResult } from "@testing-library/react"; import React from "react"; import { IntlProvider } from "react-intl"; import { MemoryRouter } from "react-router-dom"; @@ -9,20 +8,14 @@ import { configContext, defaultConfig } from "config"; import { FeatureService } from "hooks/services/Feature"; import en from "locales/en.json"; -export type RenderOptions = { - // optionally pass in a history object to control routes in the test - history?: History; - container?: HTMLElement; -}; - type WrapperProps = { - children?: React.ReactNode; + children?: React.ReactElement; }; -export async function render( - ui: React.ReactNode, - renderOptions?: RenderOptions -): Promise> { +export async function render< + Q extends Queries = typeof queries, + Container extends Element | DocumentFragment = HTMLElement +>(ui: React.ReactNode, renderOptions?: RenderOptions): Promise> { function Wrapper({ children }: WrapperProps) { return ( @@ -35,9 +28,9 @@ export async function render( ); } - let renderResult: RenderResult; + let renderResult: RenderResult; await act(async () => { - renderResult = await rtlRender(
{ui}
, { wrapper: Wrapper, ...renderOptions }); + renderResult = await rtlRender(
{ui}
, { wrapper: Wrapper, ...renderOptions }); }); return renderResult!; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx index ed9eb24f09e6cb..7c69f2b78827f6 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx @@ -28,6 +28,7 @@ import { flatten, getPathType } from "./utils"; const Section = styled.div<{ error?: boolean; isSelected: boolean }>` border: 1px solid ${(props) => (props.error ? props.theme.dangerColor : "none")}; background: ${({ theme, isSelected }) => (isSelected ? "rgba(97, 94, 255, 0.1);" : theme.greyColor0)}; + padding: 2px; &:first-child { border-radius: 8px 8px 0 0; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/StreamFieldTable.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/StreamFieldTable.tsx index 6776794f4cda55..6e5454aaf1b7a3 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/StreamFieldTable.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/StreamFieldTable.tsx @@ -38,7 +38,7 @@ export const StreamFieldTable: React.FC = (props) => { {props.syncSchemaFields.map((field) => ( - + +const useInitialSchema = (schema: SyncSchema, supportedDestinationSyncModes: DestinationSyncMode[]): SyncSchema => useMemo( () => ({ streams: schema.streams.map((apiNode, id) => { const nodeWithId: SyncSchemaStream = { ...apiNode, id: id.toString() }; + const nodeStream = verifyConfigCursorField(verifySupportedSyncModes(nodeWithId)); - // If the value in supportedSyncModes is empty assume the only supported sync mode is FULL_REFRESH. - // Otherwise, it supports whatever sync modes are present. - const streamNode = nodeWithId.stream.supportedSyncModes?.length - ? nodeWithId - : setIn(nodeWithId, "stream.supportedSyncModes", [SyncMode.FullRefresh]); - - // If syncMode isn't null - don't change item - if (streamNode.config.syncMode) { - return streamNode; - } - - const updateStreamConfig = (config: Partial): SyncSchemaStream => ({ - ...streamNode, - config: { ...streamNode.config, ...config }, - }); - - const supportedSyncModes = streamNode.stream.supportedSyncModes; - - // Prefer INCREMENTAL sync mode over other sync modes - if (supportedSyncModes.includes(SyncMode.Incremental)) { - return updateStreamConfig({ - cursorField: streamNode.config.cursorField.length - ? streamNode.config.cursorField - : getDefaultCursorField(streamNode), - syncMode: SyncMode.Incremental, - }); - } - - // If source don't support INCREMENTAL and FULL_REFRESH - set first value from supportedSyncModes list - return updateStreamConfig({ - syncMode: streamNode.stream.supportedSyncModes[0], - }); + return getOptimalSyncMode(nodeStream, supportedDestinationSyncModes); }), }), - [schema.streams] + [schema.streams, supportedDestinationSyncModes] ); const getInitialTransformations = (operations: Operation[]): Transformation[] => operations.filter(isDbtTransformation); @@ -266,7 +224,7 @@ const useInitialValues = ( destDefinition: DestinationDefinitionSpecification, isEditMode?: boolean ): FormikConnectionFormValues => { - const initialSchema = useInitialSchema(connection.syncCatalog); + const initialSchema = useInitialSchema(connection.syncCatalog, destDefinition.supportedDestinationSyncModes); return useMemo(() => { const initialValues: FormikConnectionFormValues = { diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.test.ts b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.test.ts new file mode 100644 index 00000000000000..5cd853bfc6ea64 --- /dev/null +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.test.ts @@ -0,0 +1,120 @@ +import { DestinationSyncMode, SyncMode, SyncSchemaStream } from "../../../core/domain/catalog"; +import { getOptimalSyncMode, verifyConfigCursorField, verifySupportedSyncModes } from "./formConfigHelpers"; + +const mockedStreamNode: SyncSchemaStream = { + stream: { + name: "test", + supportedSyncModes: [], + jsonSchema: {}, + sourceDefinedCursor: null, + sourceDefinedPrimaryKey: [], + defaultCursorField: [], + }, + config: { + cursorField: [], + primaryKey: [], + selected: true, + syncMode: SyncMode.FullRefresh, + destinationSyncMode: DestinationSyncMode.Append, + aliasName: "", + }, + id: "1", +}; + +describe("formConfigHelpers", () => { + describe("verifySupportedSyncModes", () => { + const streamNodeWithDefinedSyncMode: SyncSchemaStream = { + ...mockedStreamNode, + stream: { ...mockedStreamNode.stream, supportedSyncModes: [SyncMode.Incremental] }, + }; + + test("should not change supportedSyncModes if it's not empty", () => { + const streamNode = verifySupportedSyncModes(streamNodeWithDefinedSyncMode); + + expect(streamNode.stream.supportedSyncModes).toStrictEqual([SyncMode.Incremental]); + }); + + test("should set default supportedSyncModes if it's empty", () => { + const streamNodeWithEmptySyncMode = { + ...streamNodeWithDefinedSyncMode, + stream: { ...mockedStreamNode.stream, supportedSyncModes: [] }, + }; + const streamNode = verifySupportedSyncModes(streamNodeWithEmptySyncMode); + + expect(streamNode.stream.supportedSyncModes).toStrictEqual([SyncMode.FullRefresh]); + }); + }); + + describe("verifyConfigCursorField", () => { + const streamWithDefinedCursorField: SyncSchemaStream = { + ...mockedStreamNode, + config: { ...mockedStreamNode.config, cursorField: ["id"] }, + stream: { ...mockedStreamNode.stream, defaultCursorField: ["name"] }, + }; + + test("should leave cursorField value as is if it's defined", () => { + const streamNode = verifyConfigCursorField(streamWithDefinedCursorField); + + expect(streamNode.config.cursorField).toStrictEqual(["id"]); + }); + + test("should set defaultCursorField if cursorField is not defined", () => { + const streamNodeWithoutDefinedCursor = { + ...streamWithDefinedCursorField, + config: { ...mockedStreamNode.config, cursorField: [] }, + }; + const streamNode = verifyConfigCursorField(streamNodeWithoutDefinedCursor); + + expect(streamNode.config.cursorField).toStrictEqual(["name"]); + }); + + test("should leave cursorField empty if defaultCursorField not defined", () => { + const streamNode = verifyConfigCursorField(mockedStreamNode); + + expect(streamNode.config.cursorField).toStrictEqual([]); + }); + }); + + describe("getOptimalSyncMode", () => { + test("should get 'Incremental(cursor defined) => Append dedup' mode", () => { + const streamNodeWithIncrDedupMode = { + ...mockedStreamNode, + stream: { ...mockedStreamNode.stream, supportedSyncModes: [SyncMode.Incremental], sourceDefinedCursor: true }, + }; + const nodeStream = getOptimalSyncMode(streamNodeWithIncrDedupMode, [DestinationSyncMode.Dedupted]); + + expect(nodeStream.config.syncMode).toBe(SyncMode.Incremental); + expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Dedupted); + }); + + test("should get 'FullRefresh => Overwrite' mode", () => { + const nodeStream = getOptimalSyncMode(mockedStreamNode, [DestinationSyncMode.Overwrite]); + + expect(nodeStream.config.syncMode).toBe(SyncMode.FullRefresh); + expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Overwrite); + }); + + test("should get 'Incremental => Append' mode", () => { + const streamNodeWithIncrAppendMode = { + ...mockedStreamNode, + stream: { ...mockedStreamNode.stream, supportedSyncModes: [SyncMode.Incremental] }, + }; + const nodeStream = getOptimalSyncMode(streamNodeWithIncrAppendMode, [DestinationSyncMode.Append]); + + expect(nodeStream.config.syncMode).toBe(SyncMode.Incremental); + expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Append); + }); + + test("should get 'FullRefresh => Append' mode", () => { + const nodeStream = getOptimalSyncMode(mockedStreamNode, [DestinationSyncMode.Append]); + + expect(nodeStream.config.syncMode).toBe(SyncMode.FullRefresh); + expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Append); + }); + test("should return untouched nodeStream", () => { + const nodeStream = getOptimalSyncMode(mockedStreamNode, []); + + expect(nodeStream).toBe(mockedStreamNode); + }); + }); +}); diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.ts b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.ts new file mode 100644 index 00000000000000..649d98fc497b98 --- /dev/null +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.ts @@ -0,0 +1,86 @@ +import { + AirbyteStreamConfiguration, + DestinationSyncMode, + SyncMode, + SyncSchemaStream, +} from "../../../core/domain/catalog"; + +const getDefaultCursorField = (streamNode: SyncSchemaStream): string[] => { + if (streamNode.stream.defaultCursorField.length) { + return streamNode.stream.defaultCursorField; + } + return streamNode.config.cursorField; +}; + +export const verifySupportedSyncModes = (streamNode: SyncSchemaStream): SyncSchemaStream => { + const { + stream: { supportedSyncModes }, + } = streamNode; + + if (supportedSyncModes?.length) return streamNode; + return { ...streamNode, stream: { ...streamNode.stream, supportedSyncModes: [SyncMode.FullRefresh] } }; +}; + +export const verifyConfigCursorField = (streamNode: SyncSchemaStream): SyncSchemaStream => { + const { config } = streamNode; + + return { + ...streamNode, + config: { + ...config, + cursorField: config.cursorField?.length ? config.cursorField : getDefaultCursorField(streamNode), + }, + }; +}; + +export const getOptimalSyncMode = ( + streamNode: SyncSchemaStream, + supportedDestinationSyncModes: DestinationSyncMode[] +): SyncSchemaStream => { + const updateStreamConfig = ( + config: Pick + ): SyncSchemaStream => ({ + ...streamNode, + config: { ...streamNode.config, ...config }, + }); + + const { + stream: { supportedSyncModes, sourceDefinedCursor }, + } = streamNode; + + if ( + supportedSyncModes.includes(SyncMode.Incremental) && + supportedDestinationSyncModes.includes(DestinationSyncMode.Dedupted) && + sourceDefinedCursor + ) { + return updateStreamConfig({ + syncMode: SyncMode.Incremental, + destinationSyncMode: DestinationSyncMode.Dedupted, + }); + } + + if (supportedDestinationSyncModes.includes(DestinationSyncMode.Overwrite)) { + return updateStreamConfig({ + syncMode: SyncMode.FullRefresh, + destinationSyncMode: DestinationSyncMode.Overwrite, + }); + } + + if ( + supportedSyncModes.includes(SyncMode.Incremental) && + supportedDestinationSyncModes.includes(DestinationSyncMode.Append) + ) { + return updateStreamConfig({ + syncMode: SyncMode.Incremental, + destinationSyncMode: DestinationSyncMode.Append, + }); + } + + if (supportedDestinationSyncModes.includes(DestinationSyncMode.Append)) { + return updateStreamConfig({ + syncMode: SyncMode.FullRefresh, + destinationSyncMode: DestinationSyncMode.Append, + }); + } + return streamNode; +}; diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 8c2b9da37ecc73..efe47ace83bc6d 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -25,7 +25,7 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list RUN apt-get update && apt-get install -y kubectl -ARG VERSION=0.36.5-alpha +ARG VERSION=0.36.6-alpha ENV APPLICATION airbyte-workers ENV VERSION ${VERSION} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java index e40f884cf93875..61c33bd6db3810 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java @@ -23,12 +23,15 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DockerProcessFactory implements ProcessFactory { private static final Logger LOGGER = LoggerFactory.getLogger(DockerProcessFactory.class); + private static final String VERSION_DELIMITER = ":"; + private static final String DOCKER_DELIMITER = "/"; private static final Path DATA_MOUNT_DESTINATION = Path.of("/data"); private static final Path LOCAL_MOUNT_DESTINATION = Path.of("/local"); @@ -114,6 +117,9 @@ public Process create(final String jobId, rebasePath(jobRoot).toString(), // rebases the job root on the job data mount "--log-driver", "none"); + final String containerName = createContainerName(imageName, jobId, attempt); + cmd.add("--name"); + cmd.add(containerName); if (networkName != null) { cmd.add("--network"); @@ -163,6 +169,26 @@ public Process create(final String jobId, } } + private static String createContainerName(final String fullImagePath, final String jobId, final int attempt) { + final var noVersion = fullImagePath.split(VERSION_DELIMITER)[0]; + + final var nameParts = noVersion.split(DOCKER_DELIMITER); + var imageName = nameParts[nameParts.length - 1]; + + final var randSuffix = RandomStringUtils.randomAlphabetic(5).toLowerCase(); + final String suffix = "sync" + "-" + jobId + "-" + attempt + "-" + randSuffix; + + var podName = imageName + "-" + suffix; + final var podNameLenLimit = 128; + if (podName.length() > podNameLenLimit) { + final var extra = podName.length() - podNameLenLimit; + imageName = imageName.substring(extra); + podName = imageName + "-" + suffix; + } + + return podName; + } + private Path rebasePath(final Path jobRoot) { final Path relativePath = workspaceRoot.relativize(jobRoot); return DATA_MOUNT_DESTINATION.resolve(relativePath); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityImpl.java index 80e387c1b0fafb..af71dd0b75f586 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityImpl.java @@ -13,7 +13,6 @@ import io.airbyte.config.Configs; import io.airbyte.config.StandardSync; import io.airbyte.config.StandardSync.Status; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.scheduler.models.Job; import io.airbyte.scheduler.models.JobStatus; @@ -25,7 +24,6 @@ import java.io.IOException; import java.util.List; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.TimeUnit; import lombok.AllArgsConstructor; @@ -50,6 +48,12 @@ public class AutoDisableConnectionActivityImpl implements AutoDisableConnectionA public AutoDisableConnectionOutput autoDisableFailingConnection(final AutoDisableConnectionActivityInput input) { if (featureFlags.autoDisablesFailingConnections()) { try { + // if connection is already inactive, no need to disable + final StandardSync standardSync = configRepository.getStandardSync(input.getConnectionId()); + if (standardSync.getStatus() == Status.INACTIVE) { + return new AutoDisableConnectionOutput(false); + } + final int maxDaysOfOnlyFailedJobs = configs.getMaxDaysOfOnlyFailedJobsBeforeConnectionDisable(); final int maxDaysOfOnlyFailedJobsBeforeWarning = maxDaysOfOnlyFailedJobs / 2; final int maxFailedJobsInARowBeforeConnectionDisableWarning = configs.getMaxFailedJobsInARowBeforeConnectionDisable() / 2; @@ -82,7 +86,7 @@ public AutoDisableConnectionOutput autoDisableFailingConnection(final AutoDisabl return new AutoDisableConnectionOutput(false); } else if (numFailures >= configs.getMaxFailedJobsInARowBeforeConnectionDisable()) { // disable connection if max consecutive failed jobs limit has been hit - disableConnection(input.getConnectionId(), lastJob); + disableConnection(standardSync, lastJob); return new AutoDisableConnectionOutput(true); } else if (numFailures == maxFailedJobsInARowBeforeConnectionDisableWarning && !warningPreviouslySentForMaxDays) { // warn if number of consecutive failures hits 50% of MaxFailedJobsInARow @@ -102,7 +106,7 @@ public AutoDisableConnectionOutput autoDisableFailingConnection(final AutoDisabl // disable connection if only failed jobs in the past maxDaysOfOnlyFailedJobs days if (firstReplicationOlderThanMaxDisableDays && noPreviousSuccess) { - disableConnection(input.getConnectionId(), lastJob); + disableConnection(standardSync, lastJob); return new AutoDisableConnectionOutput(true); } @@ -172,8 +176,7 @@ private int getDaysSinceTimestamp(final long currentTimestampInSeconds, final lo return Math.toIntExact(TimeUnit.SECONDS.toDays(currentTimestampInSeconds - timestampInSeconds)); } - private void disableConnection(final UUID connectionId, final Job lastJob) throws JsonValidationException, IOException, ConfigNotFoundException { - final StandardSync standardSync = configRepository.getStandardSync(connectionId); + private void disableConnection(final StandardSync standardSync, final Job lastJob) throws JsonValidationException, IOException { standardSync.setStatus(Status.INACTIVE); configRepository.writeStandardSync(standardSync); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityTest.java index 36bc505899aec5..6423f2d8b484de 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/AutoDisableConnectionActivityTest.java @@ -82,7 +82,8 @@ class AutoDisableConnectionActivityTest { private final StandardSync standardSync = new StandardSync(); @BeforeEach - void setUp() throws IOException { + void setUp() throws IOException, JsonValidationException, ConfigNotFoundException { + Mockito.when(mConfigRepository.getStandardSync(CONNECTION_ID)).thenReturn(standardSync); standardSync.setStatus(Status.ACTIVE); Mockito.when(mFeatureFlags.autoDisablesFailingConnections()).thenReturn(true); Mockito.when(mConfigs.getMaxDaysOfOnlyFailedJobsBeforeConnectionDisable()).thenReturn(MAX_DAYS_OF_ONLY_FAILED_JOBS); diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index ace0f2c1c0d490..fbcf3833152504 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -21,7 +21,7 @@ version: 0.3.1 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.36.5-alpha" +appVersion: "0.36.6-alpha" dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index e7bbf27c282391..22c78bede3fa54 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -31,7 +31,7 @@ Helm charts for Airbyte. | `webapp.replicaCount` | Number of webapp replicas | `1` | | `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | | `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | -| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.36.5-alpha` | +| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.36.6-alpha` | | `webapp.podAnnotations` | Add extra annotations to the webapp pod(s) | `{}` | | `webapp.containerSecurityContext` | Security context for the container | `{}` | | `webapp.livenessProbe.enabled` | Enable livenessProbe on the webapp | `true` | @@ -73,7 +73,7 @@ Helm charts for Airbyte. | `scheduler.replicaCount` | Number of scheduler replicas | `1` | | `scheduler.image.repository` | The repository to use for the airbyte scheduler image. | `airbyte/scheduler` | | `scheduler.image.pullPolicy` | the pull policy to use for the airbyte scheduler image | `IfNotPresent` | -| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.36.5-alpha` | +| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.36.6-alpha` | | `scheduler.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | | `scheduler.resources.limits` | The resources limits for the scheduler container | `{}` | | `scheduler.resources.requests` | The requested resources for the scheduler container | `{}` | @@ -120,7 +120,7 @@ Helm charts for Airbyte. | `server.replicaCount` | Number of server replicas | `1` | | `server.image.repository` | The repository to use for the airbyte server image. | `airbyte/server` | | `server.image.pullPolicy` | the pull policy to use for the airbyte server image | `IfNotPresent` | -| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.36.5-alpha` | +| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.36.6-alpha` | | `server.podAnnotations` | Add extra annotations to the server pod | `{}` | | `server.containerSecurityContext` | Security context for the container | `{}` | | `server.livenessProbe.enabled` | Enable livenessProbe on the server | `true` | @@ -158,7 +158,7 @@ Helm charts for Airbyte. | `worker.replicaCount` | Number of worker replicas | `1` | | `worker.image.repository` | The repository to use for the airbyte worker image. | `airbyte/worker` | | `worker.image.pullPolicy` | the pull policy to use for the airbyte worker image | `IfNotPresent` | -| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.36.5-alpha` | +| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.36.6-alpha` | | `worker.podAnnotations` | Add extra annotations to the worker pod(s) | `{}` | | `worker.containerSecurityContext` | Security context for the container | `{}` | | `worker.livenessProbe.enabled` | Enable livenessProbe on the worker | `true` | @@ -190,7 +190,7 @@ Helm charts for Airbyte. | ----------------------------- | -------------------------------------------------------------------- | -------------------- | | `bootloader.image.repository` | The repository to use for the airbyte bootloader image. | `airbyte/bootloader` | | `bootloader.image.pullPolicy` | the pull policy to use for the airbyte bootloader image | `IfNotPresent` | -| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.36.5-alpha` | +| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.36.6-alpha` | ### Temporal parameters diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 1e445caba5dfa2..ad5ec07f8fcb67 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -43,7 +43,7 @@ webapp: image: repository: airbyte/webapp pullPolicy: IfNotPresent - tag: 0.36.5-alpha + tag: 0.36.6-alpha ## @param webapp.podAnnotations [object] Add extra annotations to the webapp pod(s) ## @@ -209,7 +209,7 @@ scheduler: image: repository: airbyte/scheduler pullPolicy: IfNotPresent - tag: 0.36.5-alpha + tag: 0.36.6-alpha ## @param scheduler.podAnnotations [object] Add extra annotations to the scheduler pod ## @@ -440,7 +440,7 @@ server: image: repository: airbyte/server pullPolicy: IfNotPresent - tag: 0.36.5-alpha + tag: 0.36.6-alpha ## @param server.podAnnotations [object] Add extra annotations to the server pod ## @@ -581,7 +581,7 @@ worker: image: repository: airbyte/worker pullPolicy: IfNotPresent - tag: 0.36.5-alpha + tag: 0.36.6-alpha ## @param worker.podAnnotations [object] Add extra annotations to the worker pod(s) ## @@ -699,7 +699,7 @@ bootloader: image: repository: airbyte/bootloader pullPolicy: IfNotPresent - tag: 0.36.5-alpha + tag: 0.36.6-alpha ## @param bootloader.podAnnotations [object] Add extra annotations to the bootloader pod ## diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/0-getting-started.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/0-getting-started.md index a1bc7b2227253a..6a813f32895a79 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/0-getting-started.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/0-getting-started.md @@ -12,6 +12,10 @@ This is a step-by-step guide for how to create an Airbyte source in Python to re All the commands below assume that `python` points to a version of python >=3.9.0. On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3`. +## Exchange Rates API Setup + +For this guide we will be making API calls to the Exchange Rates API. In order to generate the API access key that will be used by the new connector, you will have to follow steps on the [Exchange Rates API](https://exchangeratesapi.io/) by signing up for the Free tier plan. Once you have an API access key, you can continue with the guide. + ## Checklist * Step 1: Create the source using the template diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/3-define-inputs.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/3-define-inputs.md index e4df89230e2016..7c30fb505fd954 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/3-define-inputs.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/3-define-inputs.md @@ -17,10 +17,14 @@ connectionSpecification: title: Python Http Tutorial Spec type: object required: + - access_key - start_date - base additionalProperties: false properties: + access_key: + type: string + description: API access key used to retrieve data from the Exchange Rates API. start_date: type: string description: Start getting data from that date. @@ -37,6 +41,7 @@ connectionSpecification: In addition to metadata, we define two inputs: +* `access_key`: The API access key used to authenticate requests to the API * `start_date`: The beginning date to start tracking currency exchange rates from * `base`: The currency whose rates we're interested in tracking diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/4-connection-checking.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/4-connection-checking.md index 367dcef39ce8fd..b07f7cc67ce204 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/4-connection-checking.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/4-connection-checking.md @@ -4,6 +4,7 @@ The second operation in the Airbyte Protocol that we'll implement is the `check` This operation verifies that the input configuration supplied by the user can be used to connect to the underlying data source. Note that this user-supplied configuration has the values described in the `spec.yaml` filled in. In other words if the `spec.yaml` said that the source requires a `username` and `password` the config object might be `{ "username": "airbyte", "password": "password123" }`. You should then implement something that returns a json object reporting, given the credentials in the config, whether we were able to connect to the source. +In order to make requests the API, we need to specify the access In our case, this is a fairly trivial check since the API requires no credentials. Instead, let's verify that the user-input `base` currency is a legitimate currency. In `source.py` we'll find the following autogenerated source: ```python @@ -37,24 +38,22 @@ Following the docstring instructions, we'll change the implementation to verify return True, None ``` -Let's test out this implementation by creating two objects: a valid and an invalid config and attempt to give them as input to the connector +Let's test out this implementation by creating two objects: a valid and an invalid config and attempt to give them as input to the connector. For this section, you will need to take the API access key generated earlier and add it to both configs. Because these configs contain secrets, we recommend storing configs which contain secrets in `secrets/config.json` because the `secrets` directory is gitignored by default. ```text -echo '{"start_date": "2021-04-01", "base": "USD"}' > sample_files/config.json -echo '{"start_date": "2021-04-01", "base": "BTC"}' > sample_files/invalid_config.json -python main.py check --config sample_files/config.json -python main.py check --config sample_files/invalid_config.json +mkdir sample_files +echo '{"start_date": "2022-04-01", "base": "USD", "access_key": }' > secrets/config.json +echo '{"start_date": "2022-04-01", "base": "BTC", "access_key": }' > secrets/invalid_config.json +python main.py check --config secrets/config.json +python main.py check --config secrets/invalid_config.json ``` You should see output like the following: ```text -> python main.py check --config sample_files/config.json +> python main.py check --config secrets/config.json {"type": "CONNECTION_STATUS", "connectionStatus": {"status": "SUCCEEDED"}} -> python main.py check --config sample_files/invalid_config.json +> python main.py check --config secrets/invalid_config.json {"type": "CONNECTION_STATUS", "connectionStatus": {"status": "FAILED", "message": "Input currency BTC is invalid. Please input one of the following currencies: {'DKK', 'USD', 'CZK', 'BGN', 'JPY'}"}} ``` - -While developing, we recommend storing configs which contain secrets in `secrets/config.json` because the `secrets` directory is gitignored by default. - diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/5-declare-schema.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/5-declare-schema.md index 3ddc5ab64ce266..59c3c663932208 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/5-declare-schema.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/5-declare-schema.md @@ -10,7 +10,7 @@ We'll begin by creating a stream to represent the data that we're pulling from t ```python class ExchangeRates(HttpStream): - url_base = "https://api.exchangeratesapi.io/" + url_base = "http://api.exchangeratesapi.io/" # Set this as a noop. primary_key = None @@ -60,7 +60,7 @@ Having created this stream in code, we'll put a file `exchange_rates.json` in th With `.json` schema file in place, let's see if the connector can now find this schema and produce a valid catalog: ```text -python main.py discover --config sample_files/config.json +python main.py discover --config secrets/config.json ``` you should see some output like: diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/6-read-data.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/6-read-data.md index 2ff984deeebd49..755118c859d57a 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/6-read-data.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/6-read-data.md @@ -36,13 +36,14 @@ Let's begin by pulling data for the last day's rates by using the `/latest` endp ```python class ExchangeRates(HttpStream): - url_base = "https://api.exchangeratesapi.io/" + url_base = "http://api.exchangeratesapi.io/" primary_key = None - def __init__(self, base: str, **kwargs): + def __init__(self, config: Mapping[str, Any], **kwargs): super().__init__() - self.base = base + self.base = config['base'] + self.access_key = config['access_key'] def path( @@ -60,8 +61,8 @@ class ExchangeRates(HttpStream): stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - # The api requires that we include the base currency as a query param so we do that in this method - return {'base': self.base} + # The api requires that we include access_key as a query param so we do that in this method + return {'access_key': self.access_key} def parse_response( self, @@ -80,14 +81,14 @@ class ExchangeRates(HttpStream): return None ``` -This may look big, but that's just because there are lots of \(unused, for now\) parameters in these methods \(those can be hidden with Python's `**kwargs`, but don't worry about it for now\). Really we just added a few lines of "significant" code: 1. Added a constructor `__init__` which stores the `base` currency to query for. 2. `return {'base': self.base}` to add the `?base=` query parameter to the request based on the `base` input by the user. 3. `return [response.json()]` to parse the response from the API to match the schema of our schema `.json` file. 4. `return "latest"` to indicate that we want to hit the `/latest` endpoint of the API to get the latest exchange rate data. +This may look big, but that's just because there are lots of \(unused, for now\) parameters in these methods \(those can be hidden with Python's `**kwargs`, but don't worry about it for now\). Really we just added a few lines of "significant" code: 1. Added a constructor `__init__` which stores the `base` currency to query for and the `access_key` used for authentication. 2. `return {'access_key': self.access_key}` to add the `?access_key=` query parameter to the request based on the `access_key` input by the user. 3. `return [response.json()]` to parse the response from the API to match the schema of our schema `.json` file. 4. `return "latest"` to indicate that we want to hit the `/latest` endpoint of the API to get the latest exchange rate data. -Let's also pass the `base` parameter input by the user to the stream class: +Let's also pass the config specified by the user to the stream class: ```python def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = NoAuth() - return [ExchangeRates(authenticator=auth, base=config['base'])] + auth = NoAuth() + return [ExchangeRates(authenticator=auth, config=config)] ``` We're now ready to query the API! @@ -95,13 +96,13 @@ We're now ready to query the API! To do this, we'll need a [ConfiguredCatalog](../../../understanding-airbyte/beginners-guide-to-catalog.md). We've prepared one [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/docs/tutorials/http_api_source_assets/configured_catalog.json) -- download this and place it in `sample_files/configured_catalog.json`. Then run: ```text - python main.py read --config sample_files/config.json --catalog sample_files/configured_catalog.json + python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` you should see some output lines, one of which is a record from the API: ```text -{"type": "RECORD", "record": {"stream": "exchange_rates", "data": {"base": "USD", "rates": {"GBP": 0.7196938353, "HKD": 7.7597848573, "IDR": 14482.4824162185, "ILS": 3.2412081092, "DKK": 6.1532478279, "INR": 74.7852709971, "CHF": 0.915763343, "MXN": 19.8439387671, "CZK": 21.3545717832, "SGD": 1.3261894911, "THB": 31.4398014067, "HRK": 6.2599917253, "EUR": 0.8274720728, "MYR": 4.0979726934, "NOK": 8.3043442284, "CNY": 6.4856433595, "BGN": 1.61836988, "PHP": 48.3516756309, "PLN": 3.770872983, "ZAR": 14.2690111709, "CAD": 1.2436905254, "ISK": 124.9482829954, "BRL": 5.4526272238, "RON": 4.0738932561, "NZD": 1.3841125362, "TRY": 8.3101365329, "JPY": 108.0182043856, "RUB": 74.9555647497, "KRW": 1111.7583781547, "USD": 1.0, "AUD": 1.2840711626, "HUF": 300.6206040546, "SEK": 8.3829540753}, "date": "2021-04-26"}, "emitted_at": 1619498062000}} +"type": "RECORD", "record": {"stream": "exchange_rates", "data": {"success": true, "timestamp": 1651129443, "base": "EUR", "date": "2022-04-28", "rates": {"AED": 3.86736, "AFN": 92.13195, "ALL": 120.627843, "AMD": 489.819318, "ANG": 1.910347, "AOA": 430.073735, "ARS": 121.119674, "AUD": 1.478877, "AWG": 1.895762, "AZN": 1.794932, "BAM": 1.953851, "BBD": 2.140212, "BDT": 91.662775, "BGN": 1.957013, "BHD": 0.396929, "BIF": 2176.669098, "BMD": 1.052909, "BND": 1.461004, "BOB": 7.298009, "BRL": 5.227798, "BSD": 1.060027, "BTC": 2.6717761e-05, "BTN": 81.165435, "BWP": 12.802036, "BYN": 3.565356, "BYR": 20637.011334, "BZD": 2.136616, "CAD": 1.349329, "CDF": 2118.452361, "CHF": 1.021627, "CLF": 0.032318, "CLP": 891.760584, "CNY": 6.953724, "COP": 4171.971894, "CRC": 701.446322, "CUC": 1.052909, "CUP": 27.902082, "CVE": 110.15345, "CZK": 24.499027, "DJF": 188.707108, "DKK": 7.441548, "DOP": 58.321493, "DZD": 152.371647, "EGP": 19.458297, "ERN": 15.793633, "ETB": 54.43729, "EUR": 1, "FJD": 2.274651, "FKP": 0.80931, "GBP": 0.839568, "GEL": 3.20611, "GGP": 0.80931, "GHS": 7.976422, "GIP": 0.80931, "GMD": 56.64554, "GNF": 9416.400803, "GTQ": 8.118402, "GYD": 221.765423, "HKD": 8.261854, "HNL": 26.0169, "HRK": 7.563467, "HTG": 115.545574, "HUF": 377.172734, "IDR": 15238.748216, "ILS": 3.489582, "IMP": 0.80931, "INR": 80.654494, "IQD": 1547.023976, "IRR": 44538.040218, "ISK": 137.457233, "JEP": 0.80931, "JMD": 163.910125, "JOD": 0.746498, "JPY": 137.331903, "KES": 121.87429, "KGS": 88.581418, "KHR": 4286.72178, "KMF": 486.443591, "KPW": 947.617993, "KRW": 1339.837191, "KWD": 0.322886, "KYD": 0.883397, "KZT": 473.770223, "LAK": 12761.755235, "LBP": 1602.661797, "LKR": 376.293562, "LRD": 159.989586, "LSL": 15.604181, "LTL": 3.108965, "LVL": 0.636894, "LYD": 5.031557, "MAD": 10.541225, "MDL": 19.593772, "MGA": 4284.002369, "MKD": 61.553251, "MMK": 1962.574442, "MNT": 3153.317641, "MOP": 8.567461, "MRO": 375.88824, "MUR": 45.165684, "MVR": 16.199478, "MWK": 865.62318, "MXN": 21.530268, "MYR": 4.594366, "MZN": 67.206888, "NAD": 15.604214, "NGN": 437.399752, "NIO": 37.965356, "NOK": 9.824365, "NPR": 129.86672, "NZD": 1.616441, "OMR": 0.405421, "PAB": 1.060027, "PEN": 4.054233, "PGK": 3.73593, "PHP": 55.075028, "PKR": 196.760944, "PLN": 4.698101, "PYG": 7246.992296, "QAR": 3.833603, "RON": 4.948144, "RSD": 117.620172, "RUB": 77.806269, "RWF": 1086.709833, "SAR": 3.949063, "SBD": 8.474149, "SCR": 14.304711, "SDG": 470.649944, "SEK": 10.367719, "SGD": 1.459695, "SHP": 1.45028, "SLL": 13082.391386, "SOS": 609.634325, "SRD": 21.904702, "STD": 21793.085136, "SVC": 9.275519, "SYP": 2645.380032, "SZL": 16.827859, "THB": 36.297991, "TJS": 13.196811, "TMT": 3.685181, "TND": 3.22348, "TOP": 2.428117, "TRY": 15.575532, "TTD": 7.202107, "TWD": 31.082183, "TZS": 2446.960099, "UAH": 32.065033, "UGX": 3773.578577, "USD": 1.052909, "UYU": 43.156886, "UZS": 11895.19696, "VEF": 225143710305.04727, "VND": 24171.62598, "VUV": 118.538204, "WST": 2.722234, "XAF": 655.287181, "XAG": 0.045404, "XAU": 0.000559, "XCD": 2.845538, "XDR": 0.783307, "XOF": 655.293398, "XPF": 118.347299, "YER": 263.490114, "ZAR": 16.77336, "ZMK": 9477.445964, "ZMW": 18.046154, "ZWL": 339.036185}}, "emitted_at": 1651130169364}} ``` There we have it - a stream which reads data in just a few lines of code! @@ -127,10 +128,10 @@ Let's get the easy parts out of the way and pass the `start_date`: ```python def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = NoAuth() - # Parse the date from a string into a datetime object - start_date = datetime.strptime(config['start_date'], '%Y-%m-%d') - return [ExchangeRates(authenticator=auth, base=config['base'], start_date=start_date)] + auth = NoAuth() + # Parse the date from a string into a datetime object + start_date = datetime.strptime(config['start_date'], '%Y-%m-%d') + return [ExchangeRates(authenticator=auth, config=config, start_date=start_date)] ``` Let's also add this parameter to the constructor and declare the `cursor_field`: @@ -141,18 +142,19 @@ from airbyte_cdk.sources.streams import IncrementalMixin class ExchangeRates(HttpStream, IncrementalMixin): - url_base = "https://api.exchangeratesapi.io/" + url_base = "http://api.exchangeratesapi.io/" cursor_field = "date" primary_key = "date" - def __init__(self, base: str, start_date: datetime, **kwargs): + def __init__(self, config: Mapping[str, Any], start_date: datetime, **kwargs): super().__init__() - self.base = base + self.base = config['base'] + self.access_key = config['access_key'] self.start_date = start_date self._cursor_value = None ``` -Declaring the `cursor_field` informs the framework that this stream now supports incremental sync. The next time you run `python main_dev.py discover --config sample_files/config.json` you'll find that the `supported_sync_modes` field now also contains `incremental`. +Declaring the `cursor_field` informs the framework that this stream now supports incremental sync. The next time you run `python main_dev.py discover --config secrets/config.json` you'll find that the `supported_sync_modes` field now also contains `incremental`. But we're not quite done with supporting incremental, we have to actually emit state! We'll structure our state object very simply: it will be a `dict` whose single key is `'date'` and value is the date of the last day we synced data from. For example, `{'date': '2021-04-26'}` indicates the connector previously read data up until April 26th and therefore shouldn't re-read anything before April 26th. @@ -226,17 +228,17 @@ We should now have a working implementation of incremental sync! Let's try it out: ```text -python main.py read --config sample_files/config.json --catalog sample_files/configured_catalog.json +python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` You should see a bunch of `RECORD` messages and `STATE` messages. To verify that incremental sync is working, pass the input state back to the connector and run it again: ```text # Save the latest state to sample_files/state.json -python main.py read --config sample_files/config.json --catalog sample_files/configured_catalog.json | grep STATE | tail -n 1 | jq .state.data > sample_files/state.json +python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json | grep STATE | tail -n 1 | jq .state.data > sample_files/state.json # Run a read operation with the latest state message -python main.py read --config sample_files/config.json --catalog sample_files/configured_catalog.json --state sample_files/state.json +python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json --state sample_files/state.json ``` You should see that only the record from the last date is being synced! This is acceptable behavior, since Airbyte requires at-least-once delivery of records, so repeating the last record twice is OK. diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 6916d66d93b03e..68d90a3e254baa 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -177,75 +177,81 @@ There are 2 available options to upload data to BigQuery `Standard` and `GCS Sta ### `GCS Staging` This is the recommended configuration for uploading data to BigQuery. It works by first uploading all the data to a [GCS](https://cloud.google.com/storage) bucket, then ingesting the data to BigQuery. To configure GCS Staging, you'll need the following parameters: + * **GCS Bucket Name** * **GCS Bucket Path** * **Block Size (MB) for GCS multipart upload** * **GCS Bucket Keep files after migration** - * See [this](https://cloud.google.com/storage/docs/creating-buckets) for instructions on how to create a GCS bucket. The bucket cannot have a retention policy. Set Protection Tools to none or Object versioning. + * See [this](https://cloud.google.com/storage/docs/creating-buckets) for instructions on how to create a GCS bucket. The bucket cannot have a retention policy. Set Protection Tools to none or Object versioning. * **HMAC Key Access ID** - * See [this](https://cloud.google.com/storage/docs/authentication/managing-hmackeys) on how to generate an access key. For more information on hmac keys please reference the [GCP docs](https://cloud.google.com/storage/docs/authentication/hmackeys) - * We recommend creating an Airbyte-specific user or service account. This user or account will require the following permissions for the bucket: - ``` - storage.multipartUploads.abort - storage.multipartUploads.create - storage.objects.create - storage.objects.delete - storage.objects.get - storage.objects.list - ``` - You can set those by going to the permissions tab in the GCS bucket and adding the appropriate the email address of the service account or user and adding the aforementioned permissions. + * See [this](https://cloud.google.com/storage/docs/authentication/managing-hmackeys) on how to generate an access key. For more information on hmac keys please reference the [GCP docs](https://cloud.google.com/storage/docs/authentication/hmackeys) + * We recommend creating an Airbyte-specific user or service account. This user or account will require the following permissions for the bucket: + ``` + storage.multipartUploads.abort + storage.multipartUploads.create + storage.objects.create + storage.objects.delete + storage.objects.get + storage.objects.list + ``` + You can set those by going to the permissions tab in the GCS bucket and adding the appropriate the email address of the service account or user and adding the aforementioned permissions. * **Secret Access Key** - * Corresponding key to the above access ID. + * Corresponding key to the above access ID. * Make sure your GCS bucket is accessible from the machine running Airbyte. This depends on your networking setup. The easiest way to verify if Airbyte is able to connect to your GCS bucket is via the check connection tool in the UI. ### `Standard` uploads + This uploads data directly from your source to BigQuery. While this is faster to setup initially, **we strongly recommend that you do not use this option for anything other than a quick demo**. It is more than 10x slower than the GCS uploading option and will fail for many datasets. Please be aware you may see some failures for big datasets and slow sources, e.g. if reading from source takes more than 10-12 hours. This is caused by the Google BigQuery SDK client limitations. For more details please check [https://github.com/airbytehq/airbyte/issues/3549](https://github.com/airbytehq/airbyte/issues/3549) ## CHANGELOG ### bigquery -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 1.1.1 | 2022-04-15 | [12068](https://github.com/airbytehq/airbyte/pull/12068) | Fixed bug with GCS bucket conditional binding | -| 1.1.0 | 2022-04-06 | [11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | -| 1.0.2 | 2022-03-30 | [11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | -| 1.0.1 | 2022-03-24 | [11350](https://github.com/airbytehq/airbyte/pull/11350) | Improve check performance | -| 1.0.0 | 2022-03-18 | [11238](https://github.com/airbytehq/airbyte/pull/11238) | Updated spec and documentation | -| 0.6.12 | 2022-03-18 | [10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters | -| 0.6.11 | 2022-03-03 | [10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM | -| 0.6.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.6.6 | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959) | Fix null pointer exception from buffered stream consumer. | -| 0.6.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | -| 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | -| 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | -| 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | -| 0.6.2 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | -| 0.6.1 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:--------------------------------------------------------------------------------------------| +| 1.1.3 | 2022-05-02 | [12528](https://github.com/airbytehq/airbyte/pull/12528/) | Update Dataset location field description | +| 1.1.2 | 2022-04-29 | [12477](https://github.com/airbytehq/airbyte/pull/12477) | Dataset location is a required field | +| 1.1.1 | 2022-04-15 | [12068](https://github.com/airbytehq/airbyte/pull/12068) | Fixed bug with GCS bucket conditional binding | +| 1.1.0 | 2022-04-06 | [11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | +| 1.0.2 | 2022-03-30 | [11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | +| 1.0.1 | 2022-03-24 | [11350](https://github.com/airbytehq/airbyte/pull/11350) | Improve check performance | +| 1.0.0 | 2022-03-18 | [11238](https://github.com/airbytehq/airbyte/pull/11238) | Updated spec and documentation | +| 0.6.12 | 2022-03-18 | [10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters | +| 0.6.11 | 2022-03-03 | [10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM | +| 0.6.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.6.6 | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959) | Fix null pointer exception from buffered stream consumer. | +| 0.6.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | +| 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | +| 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | +| 0.6.2 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | +| 0.6.1 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging | | 0.6.0 | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/issues/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files | -| 0.5.1 | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations | -| 0.5.0 | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables | -| 0.4.1 | 2021-10-04 | [\#6733](https://github.com/airbytehq/airbyte/issues/6733) | Support dataset starting with numbers | -| 0.4.0 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added GCS Staging uploading option | -| 0.3.12 | 2021-08-03 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add optional arg to make a possibility to change the BigQuery client's chunk\buffer size | -| 0.3.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.3.10 | 2021-07-28 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add extended logs and made JobId filled with region and projectId | -| 0.3.9 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | -| 0.3.6 | 2021-06-18 | [\#3947](https://github.com/airbytehq/airbyte/issues/3947) | Service account credentials are now optional. | -| 0.3.4 | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option | +| 0.5.1 | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations | +| 0.5.0 | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables | +| 0.4.1 | 2021-10-04 | [\#6733](https://github.com/airbytehq/airbyte/issues/6733) | Support dataset starting with numbers | +| 0.4.0 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added GCS Staging uploading option | +| 0.3.12 | 2021-08-03 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add optional arg to make a possibility to change the BigQuery client's chunk\buffer size | +| 0.3.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.3.10 | 2021-07-28 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add extended logs and made JobId filled with region and projectId | +| 0.3.9 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | +| 0.3.6 | 2021-06-18 | [\#3947](https://github.com/airbytehq/airbyte/issues/3947) | Service account credentials are now optional. | +| 0.3.4 | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option | ### bigquery-denormalized | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------| :--- | -| 0.3.1 | 2022-04-15 | [11978](https://github.com/airbytehq/airbyte/pull/11978) | Fixed emittedAt timestamp. | -| 0.3.0 | 2022-04-06 | [11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | -| 0.2.15 | 2022-04-05 | [11166](https://github.com/airbytehq/airbyte/pull/11166) | Fixed handling of anyOf and allOf fields | -| 0.2.14 | 2022-04-02 | [11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | -| 0.2.13 | 2022-04-01 | [11636](https://github.com/airbytehq/airbyte/pull/11636) | Added new unit tests | -| 0.2.12 | 2022-03-28 | [11454](https://github.com/airbytehq/airbyte/pull/11454) | Integration test enhancement for picking test-data and schemas | -| 0.2.11 | 2022-03-18 | [10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters | -| 0.2.10 | 2022-03-03 | [10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM | +| 0.3.3 | 2022-05-02 | [12528](https://github.com/airbytehq/airbyte/pull/12528/) | Update Dataset location field description | +| 0.3.2 | 2022-04-29 | [12477](https://github.com/airbytehq/airbyte/pull/12477) | Dataset location is a required field | +| 0.3.1 | 2022-04-15 | [11978](https://github.com/airbytehq/airbyte/pull/11978) | Fixed emittedAt timestamp. | +| 0.3.0 | 2022-04-06 | [11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | +| 0.2.15 | 2022-04-05 | [11166](https://github.com/airbytehq/airbyte/pull/11166) | Fixed handling of anyOf and allOf fields | +| 0.2.14 | 2022-04-02 | [11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | +| 0.2.13 | 2022-04-01 | [11636](https://github.com/airbytehq/airbyte/pull/11636) | Added new unit tests | +| 0.2.12 | 2022-03-28 | [11454](https://github.com/airbytehq/airbyte/pull/11454) | Integration test enhancement for picking test-data and schemas | +| 0.2.11 | 2022-03-18 | [10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters | +| 0.2.10 | 2022-03-03 | [10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM | | 0.2.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.2.7 | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959) | Fix null pointer exception from buffered stream consumer. | | 0.2.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index facd3f56c66b56..1e4552aac1dfb4 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -1,36 +1,99 @@ # S3 -## Features +This page guides you through the process of setting up the S3 destination connector. -| Feature | Support | Notes | -| :--- | :---: | :--- | -| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | -| Incremental - Append Sync | ✅ | | -| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. | -| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | +## Prerequisites -The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket. - -## Troubleshooting - -Check out common troubleshooting issues for the S3 destination connector on our Discourse [here](https://discuss.airbyte.io/tags/c/connector/11/destination-s3). +List of required fields: +* **Access Key ID** +* **Secret Access Key** +* **S3 Bucket Name** +* **S3 Bucket Path** +* **S3 Bucket Region** -## Configuration +1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). +2. An S3 bucket with credentials or an instanceprofile with read/write permissions configured for the host (ec2, eks). -| Parameter | Type | Notes | -| :--- | :---: | :--- | -| S3 Endpoint | string | URL to S3, If using AWS S3 just leave blank. | -| S3 Bucket Name | string | Name of the bucket to sync data into. | -| S3 Bucket Path | string | Subdirectory under the above bucket to sync the data into. | -| S3 Bucket Format | string | Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. | -| S3 Region | string | See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. | -| Access Key ID | string | AWS/Minio credential. | -| Secret Access Key | string | AWS/Minio credential. | -| Format | object | Format specific configuration. See the [spec](/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json) for details. | +## Step 1: Set up S3 + +[Sign in](https://signin.aws.amazon.com/signin) to your AWS account. +Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). + +Prepare S3 bucket that will be used as destination, see [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + +## Step 2: Set up the S3 destination connector in Airbyte + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +4. Configure fields: + * **Access Key Id** + * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. + * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket. + * **Secret Access Key** + * Corresponding key to the above key id. + * **S3 Bucket Name** + * See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + * **S3 Bucket Path** + * Subdirectory under the above bucket to sync the data into. + * **S3 Bucket Region** + * See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + * **S3 Path Format** + * Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + * **S3 Endpoint** + * Leave empty if using AWS S3, fill in S3 URL if using Minio S3. +5. Click `Set up destination`. + +**For Airbyte OSS:** + +1. Go to local Airbyte page. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +4. Configure fields: + * **Access Key Id** + * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. + * See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. + * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. + * If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. + * **Secret Access Key** + * Corresponding key to the above key id. + * Make sure your S3 bucket is accessible from the machine running Airbyte. + * This depends on your networking setup. + * You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). + * If you use instance profile authentication, make sure the role has permission to read/write on the bucket. + * The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. + * **S3 Bucket Name** + * See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + * **S3 Bucket Path** + * Subdirectory under the above bucket to sync the data into. + * **S3 Bucket Region** + * See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + * **S3 Path Format** + * Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + * **S3 Endpoint** + * Leave empty if using AWS S3, fill in S3 URL if using Minio S3. +5. Click `Set up destination`. -⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use: +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME/*", + "arn:aws:s3:::YOUR_BUCKET_NAME" + ] + } + ] +} +``` -The full path of the output data with the default S3 path format `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: +The full path of the output data with the default S3 Path Format `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: ```text ///__. @@ -70,14 +133,26 @@ But it is possible to further customize by using the available variables to form - `${EPOCH}`: Milliseconds since Epoch in which the sync was writing the output data in. - `${UUID}`: random uuid string -Note: -- Multiple `/` characters in the S3 path are collapsed into a single `/` character. +Note: +- Multiple `/` characters in the S3 path are collapsed into a single `/` character. - If the output bucket contains too many files, the part id variable is using a `UUID` instead. It uses sequential ID otherwise. Please note that the stream name may contain a prefix, if it is configured on the connection. -A data sync may create multiple files as the output files can be partitioned by size (targeting a size of 200MB compressed or lower) . +A data sync may create multiple files as the output files can be partitioned by size (targeting a size of 200MB compressed or lower) . -## Output Schema +## Supported sync modes + +| Feature | Support | Notes | +| :--- | :---: | :--- | +| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | +| Incremental - Append Sync | ✅ | | +| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. | +| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | + +The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket. +⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ + +## Supported Output schema Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world. @@ -117,7 +192,7 @@ Here is the available compression codecs: #### Data schema -Under the hood, an Airbyte data stream in Json schema is first converted to an Avro schema, then the Json object is converted to an Avro record. Because the data stream can come from any data source, the Json to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.io/understanding-airbyte/json-avro-conversion). +Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the JSON object is converted to an Avro record. Because the data stream can come from any data source, the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.io/understanding-airbyte/json-avro-conversion). ### CSV @@ -158,7 +233,7 @@ Output files can be compressed. The default option is GZIP compression. If compr ### JSON Lines \(JSONL\) -[Json Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a structure as follows: +[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a structure as follows: ```json { @@ -217,35 +292,7 @@ These parameters are related to the `ParquetOutputFormat`. See the [Java doc](ht #### Data schema -Under the hood, an Airbyte data stream in Json schema is first converted to an Avro schema, then the Json object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the Json to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.io/understanding-airbyte/json-avro-conversion). - -## Getting Started \(Airbyte Open-Source / Airbyte Cloud\) - -#### Requirements - -1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials or an instanceprofile with read/write permissions configured for the host (ec2, eks). - -#### Setup Guide - -* Fill up S3 info - * **S3 Endpoint** - * Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - * **S3 Bucket Name** - * See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. - * **S3 Bucket Region** - * **Access Key Id** - * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - * See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. - * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. - * If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. - * **Secret Access Key** - * Corresponding key to the above key id. -* Make sure your S3 bucket is accessible from the machine running Airbyte. - * This depends on your networking setup. - * You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). - * If you will use instance profile authentication, make sure the role has permission to read/write on the bucket. - * The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. +Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the JSON object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.io/understanding-airbyte/json-avro-conversion). In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use: ```json diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index d31a5571f4cba9..3de1b54f3b0f3f 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- | :--- | +| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | | 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | @@ -110,6 +111,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.8 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.1.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | @@ -118,4 +120,3 @@ Your database user should now be ready for use with Airbyte. | 0.1.2 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery | | 0.1.1 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | | 0.1.0 | 2021-11-23 | [7457](https://github.com/airbytehq/airbyte/pull/7457) | CockroachDb source: Add only encrypted version for the connector | - diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index 173f7973d94a66..7a029ecb619ed5 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -62,6 +62,7 @@ You can also enter your own password for the keystore, but if you don't, the pas | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |**** @@ -72,4 +73,3 @@ You can also enter your own password for the keystore, but if you don't, the pas | 0.1.2 | 2021-10-25 | [7355](https://github.com/airbytehq/airbyte/pull/7355) | Added ssl support | | 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | | 0.1.0 | 2021-06-22 | [4197](https://github.com/airbytehq/airbyte/pull/4197) | New Source: IBM DB2 | - diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index 8b6259b235a38a..ce17f977fe8127 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -63,6 +63,7 @@ GitLab source is working with GitLab API v4. It can also work with self-hosted G | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------| :--- | +| 0.1.5 | 2022-05-02 | [11907](https://github.com/airbytehq/airbyte/pull/11907) | Fix null projects param and `container_expiration_policy` | | 0.1.4 | 2022-03-23 | [11140](https://github.com/airbytehq/airbyte/pull/11140) | Ingest All Accessible Groups if not Specified in Config | | 0.1.3 | 2021-12-21 | [8991](https://github.com/airbytehq/airbyte/pull/8991) | Update connector fields title/description | | 0.1.2 | 2021-10-18 | [7108](https://github.com/airbytehq/airbyte/pull/7108) | Allow all domains to be used as `api_url` | diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 353218e67d6142..b4e38c7966a65f 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -147,12 +147,13 @@ If you are using OAuth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------| -| 0.1.55 | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream | -| 0.1.54 | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s | -| 0.1.53 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | chaneg spec json to yaml format | +| 0.1.56 | 2022-05-02 | [12515](https://github.com/airbytehq/airbyte/pull/12515) | Extra logs for troubleshooting 403 errors | +| 0.1.55 | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream | +| 0.1.54 | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s | +| 0.1.53 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | chaneg spec json to yaml format | | 0.1.52 | 2022-03-25 | [11423](https://github.com/airbytehq/airbyte/pull/11423) | Add tickets associations to engagements streams | -| 0.1.51 | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue | -| 0.1.50 | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination | +| 0.1.51 | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue | +| 0.1.50 | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination | | 0.1.49 | 2022-03-17 | [11218](https://github.com/airbytehq/airbyte/pull/11218) | Anchor hyperlink in input configuration | | 0.1.48 | 2022-03-16 | [11105](https://github.com/airbytehq/airbyte/pull/11105) | Fix float numbers, upd docs | | 0.1.47 | 2022-03-15 | [11121](https://github.com/airbytehq/airbyte/pull/11121) | Add partition keys where appropriate | diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 974d744ecb42c7..898e67767619a2 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -1,54 +1,17 @@ # Instagram -## Sync overview +This page guides you through the process of setting up the instagram source connector. -This source can sync data for the Instagram Business Account available in the Facebook Graph API: User, Media, and Stories. It can also sync Media/Story and User Insights. -### Output schema - -This Source is capable of syncing the following core Streams: - -* [User](https://developers.facebook.com/docs/instagram-api/reference/ig-user) - * [User Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights) -* [Media](https://developers.facebook.com/docs/instagram-api/reference/ig-user/media) - * [Media Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights) -* [Stories](https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories/) - * [Story Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights) - -For more information, see the [Instagram API](https://developers.facebook.com/docs/instagram-api/) and [Instagram Insights API documentation](https://developers.facebook.com/docs/instagram-api/guides/insights/). - -### Data type mapping - -| Integration Type | Airbyte Type | -| :--- | :--- | -| `string` | `string` | -| `number` | `number` | -| `array` | `array` | -| `object` | `object` | - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | only User Insights | - -### Rate Limiting & Performance Considerations - -Instagram, like all Facebook services, has a limit on the number of requests, but Instagram connector gracefully handles rate limiting. - -See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting/#instagram-graph-api) for more information. - -## Getting started - -### Requirements +## Prerequisites * A Facebook App * An Instagram Business Account * A Facebook Page linked to your Instagram Business Account * A Facebook API Access Token -### Setup guide + +## Step 1: Set up Instagram ### Facebook App @@ -64,19 +27,75 @@ See the Facebook [support](https://www.facebook.com/business/help/89875296019580 Follow the [Instagram documentation](https://www.facebook.com/business/help/1492627900875762) for setting up an Instagram business account. We'll need this ID to configure Instagram as a source in Airbyte. -### API Access Token -To work with the Instagram connector, you need to generate an Access Token with the following permissions: +## Step 2: Set up the Instagram connector in Airbyte + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **Instagram** from the Source type dropdown and enter a name for this connector. +4. Select `Authenticate your account`. +5. Log in and Authorize to the Instagram account and click `Set up source`. + +**For Airbyte OSS:** +1. For using an Access Tokens, set up instagram (see step above). +2. Generate [Access Tokens](https://developers.facebook.com/docs/facebook-login/access-tokens/#usertokens) with the following permissions: * [instagram\_basic](https://developers.facebook.com/docs/permissions/reference/instagram_basic) * [instagram\_manage\_insights](https://developers.facebook.com/docs/permissions/reference/instagram_manage_insights) * [pages\_show\_list](https://developers.facebook.com/docs/permissions/reference/pages_show_list) * [pages\_read\_engagement](https://developers.facebook.com/docs/permissions/reference/pages_read_engagement) * [Instagram Public Content Access](https://developers.facebook.com/docs/apps/features-reference/instagram-public-content-access) +3. Go to local Airbyte page. +4. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +5. On the Set up the source page, enter the name for the Instagram connector and select **Instagram** from the Source type dropdown. +6. Paste your Access Tokens from step 2. +7. Click `Set up source`. -More details how to get a User's Access Token you can find in the following docs: [Access Tokens](https://developers.facebook.com/docs/facebook-login/access-tokens/#usertokens) and [User and Page Access Tokens](https://developers.facebook.com/docs/pages/access-tokens) -With the Instagram Account ID and API access token, you should be ready to start pulling data from the Facebook Instagram API. Head to the Airbyte UI to setup your source connector! +## Supported sync modes + +The Instagram source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + - Incremental + + +## Supported Streams + +* [User](https://developers.facebook.com/docs/instagram-api/reference/ig-user) + * [User Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights) +* [Media](https://developers.facebook.com/docs/instagram-api/reference/ig-user/media) + * [Media Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights) +* [Stories](https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories/) + * [Story Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights) + +For more information, see the [Instagram API](https://developers.facebook.com/docs/instagram-api/) and [Instagram Insights API documentation](https://developers.facebook.com/docs/instagram-api/guides/insights/). + + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | only User Insights | + +### Rate Limiting & Performance Considerations + +Instagram, like all Facebook services, has a limit on the number of requests, but Instagram connector gracefully handles rate limiting. + +See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting/#instagram-graph-api) for more information. + + +## Data type map + +| Integration Type | Airbyte Type | +| :--- | :--- | +| `string` | `string` | +| `number` | `number` | +| `array` | `array` | +| `object` | `object` | + ## Changelog @@ -86,4 +105,3 @@ With the Instagram Account ID and API access token, you should be ready to start | 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | added check for empty state and fixed tests. | | 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous format of STATE. | | 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling. - fix sync fail with HTTP status 400. - integrate SAT. | - diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index f1f2eb8aa82993..c2a87a83253211 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -60,6 +60,7 @@ Please read [How to get your Access Token](https://developers.intercom.com/build | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.17 | 2022-04-29 | [12374](https://github.com/airbytehq/airbyte/pull/12374) | Fixed filtering of conversation_parts | | 0.1.16 | 2022-03-23 | [11206](https://github.com/airbytehq/airbyte/pull/11206) | Added conversation_id field to conversation_part records | | 0.1.15 | 2022-03-22 | [11176](https://github.com/airbytehq/airbyte/pull/11176) | Correct `check_connection` URL | | 0.1.14 | 2022-03-16 | [11208](https://github.com/airbytehq/airbyte/pull/11208) | Improve 'conversations' incremental sync speed | diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index debb8bb54f0295..4bbb274e0e898b 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -59,6 +59,7 @@ Select the correct region \(EU or US\) for your Mixpanel project. See detail [he | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| `0.1.14` | 2022-05-02 | [11501](https://github.com/airbytehq/airbyte/pull/11501) | Improve incremental sync method to streams | | `0.1.13` | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | | `0.1.12` | 2022-03-31 | [11633](https://github.com/airbytehq/airbyte/pull/11633) | Increase unit test coverage | | `0.1.11` | 2022-04-04 | [11318](https://github.com/airbytehq/airbyte/pull/11318) | Change Response Reading | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 69495aec355606..43790f189caf48 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -294,6 +294,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------------------------------------------------- | :------------------------------------- | +| 0.3.22 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.3.21 | 2022-04-11 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.3.19 | 2022-03-31 | [11495](https://github.com/airbytehq/airbyte/pull/11495) | Adds Support to Chinese MSSQL Server Agent | | 0.3.18 | 2022-03-29 | [11010](https://github.com/airbytehq/airbyte/pull/11010) | Adds JDBC Params | @@ -326,4 +327,3 @@ If you do not see a type in this list, assume that it is coerced into a string. | 0.1.6 | 2020-12-09 | [1172](https://github.com/airbytehq/airbyte/pull/1172) | Support incremental sync | | | 0.1.5 | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038) | Change JDBC sources to discover more than standard schemas | | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | | - diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index e9d8298526d00c..811c26f111f986 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -185,6 +185,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| +| 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.5.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.5.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 0b2432879f37f1..031e55def8e218 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -132,6 +132,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:------------------------------------------------| +| 0.3.15 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.3.14 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.3.13 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.3.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | @@ -145,4 +146,3 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | 0.3.4 | 2021-09-01 | [6038](https://github.com/airbytehq/airbyte/pull/6038) | Remove automatic filtering of system schemas. | | 0.3.3 | 2021-09-01 | [5779](https://github.com/airbytehq/airbyte/pull/5779) | Ability to only discover certain schemas. | | 0.3.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator. | - diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 6b349289c48003..f5220081c57f5c 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -270,6 +270,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.4.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.4.11 | 2022-04-11 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.4.10 | 2022-04-08 | [11798](https://github.com/airbytehq/airbyte/pull/11798) | Fixed roles for fetching materialized view processing | | 0.4.8 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | @@ -313,4 +314,3 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | 0.1.6 | 2020-12-09 | [1172](https://github.com/airbytehq/airbyte/pull/1172) | Support incremental sync | | 0.1.5 | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038) | Change JDBC sources to discover more than standard schemas | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | - diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index 34ca674590756a..33bd04918bc269 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -54,6 +54,7 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |0 | 0.3.9 | 2022-02-21 | [9744](https://github.com/airbytehq/airbyte/pull/9744) | List only the tables on which the user has SELECT permissions. | 0.3.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.3.7 | 2022-01-26 | [9721](https://github.com/airbytehq/airbyte/pull/9721) | Added schema selection | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index f500081c8b2a63..ddeca9ac1d0352 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -103,6 +103,7 @@ Field | Description | | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.11 | 2022-04-27 | [10953](https://github.com/airbytehq/airbyte/pull/10953) | Implement OAuth flow | | 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index 93e6864d77be30..5cea0d66f7331b 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -120,4 +120,5 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | | :------ | :--- | :----------- | ------- | +| 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release | diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index 9ae8455bfcad27..bc8aaef785493c 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -3,6 +3,8 @@ This page guides you through the process of setting up the TikTok Marketing source connector. ## Prerequisites +* Start date +* Report Granularity (LIFETIME, DAY, HOUR) For Production environment: * Access token @@ -13,8 +15,6 @@ For Sandbox environment: * Access token * Advertiser ID -* Start date -* Report Granularity (LIFETIME, DAY, HOUR) ## Step 1: Set up TikTok @@ -521,6 +521,7 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| +| 0.1.8 | 2022-04-28 | [12435](https://github.com/airbytehq/airbyte/pull/12435) | updated spec descriptions | | 0.1.7 | 2022-04-27 | [12380](https://github.com/airbytehq/airbyte/pull/12380) | fixed spec descriptions and documentation | | 0.1.6 | 2022-04-19 | [11378](https://github.com/airbytehq/airbyte/pull/11378) | updated logic for stream initializations, fixed errors in schemas, updated SAT and unit tests | | 0.1.5 | 2022-02-17 | [10398](https://github.com/airbytehq/airbyte/pull/10398) | Add Audience reports | diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 10d152049f4af1..acd9582b16dc1b 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -103,7 +103,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.36.5-alpha --\ + docker run --rm -v /tmp:/config airbyte/migration:0.36.6-alpha --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index f648fced94bb1d..1b6354b471841b 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.36.5-alpha +AIRBYTE_VERSION=0.36.6-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml index 7a0998ce279034..7e2fca766fe400 100644 --- a/kube/overlays/stable-with-resource-limits/kustomization.yaml +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -8,17 +8,17 @@ bases: images: - name: airbyte/db - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/bootloader - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/scheduler - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/server - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/webapp - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/worker - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 2a231f777ab9b6..a3fe3a2a989146 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.36.5-alpha +AIRBYTE_VERSION=0.36.6-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index a59991df33dea2..d68e6d49ba26f4 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -8,17 +8,17 @@ bases: images: - name: airbyte/db - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/bootloader - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/scheduler - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/server - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/webapp - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: airbyte/worker - newTag: 0.36.5-alpha + newTag: 0.36.6-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index 291e9dbd7da3f1..75899e30eeebb5 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -14,5 +14,5 @@ USER octavia-cli WORKDIR /home/octavia-project ENTRYPOINT ["octavia"] -LABEL io.airbyte.version=0.36.5-alpha +LABEL io.airbyte.version=0.36.6-alpha LABEL io.airbyte.name=airbyte/octavia-cli diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 2de64af914cc02..491a7025942ac6 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -105,7 +105,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.36.5-alpha +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.36.6-alpha ``` ### Using `docker-compose` diff --git a/octavia-cli/install.sh b/octavia-cli/install.sh index 032a375959f2b4..66da8b26d01e19 100755 --- a/octavia-cli/install.sh +++ b/octavia-cli/install.sh @@ -3,7 +3,7 @@ # This install scripts currently only works for ZSH and Bash profiles. # It creates an octavia alias in your profile bound to a docker run command and your current user. -VERSION=0.36.5-alpha +VERSION=0.36.6-alpha OCTAVIA_ENV_FILE=${HOME}/.octavia detect_profile() { diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index f50f51bb63ea1f..dcb27719e6b6c3 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -15,7 +15,7 @@ setup( name="octavia-cli", - version="0.36.5", + version="0.36.6", description="A command line interface to manage Airbyte configurations", long_description=README, author="Airbyte", diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index fdb40bb8b3b257..719bfd7ef78d77 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -10,6 +10,7 @@ Usage: $(basename "$0") For publish, if you want to push the spec to the spec cache, provide a path to a service account key file that can write to the cache. Available commands: scaffold + test build [] publish [] [--publish_spec_to_cache] [--publish_spec_to_cache_with_key_file ] publish_external @@ -51,6 +52,117 @@ cmd_build() { fi } +# Experimental version of the above for a new way to build/tag images +cmd_build_experiment() { + local path=$1; shift || error "Missing target (root path of integration) $USAGE" + [ -d "$path" ] || error "Path must be the root path of the integration" + + echo "Building $path" + ./gradlew --no-daemon "$(_to_gradle_path "$path" clean)" + ./gradlew --no-daemon "$(_to_gradle_path "$path" build)" + + # After this happens this image should exist: "image_name:dev" + # Re-tag with CI candidate label + local image_name; image_name=$(_get_docker_image_name "$path/Dockerfile") + local image_version; image_version=$(_get_docker_image_version "$path/Dockerfile") + local image_candidate_tag; image_candidate_tag="$image_version-candidate-$PR_NUMBER" + + # If running via the bump-build-test-connector job, re-tag gradle built image following candidate image pattern + if [[ "$GITHUB_JOB" == "bump-build-test-connector" ]]; then + docker tag "$image_name:dev" "$image_name:$image_candidate_tag" + # TODO: docker push "$image_name:$image_candidate_tag" + fi +} + +cmd_test() { + local path=$1; shift || error "Missing target (root path of integration) $USAGE" + [ -d "$path" ] || error "Path must be the root path of the integration" + + # TODO: needs to know to use alternate image tag from cmd_build_experiment + echo "Running integration tests..." + ./gradlew --no-daemon "$(_to_gradle_path "$path" integrationTest)" +} + +# Bumps connector version in Dockerfile, definitions.yaml file, and updates seeds with gradle. +# This does not build or test, it solely manages the versions of connectors to be +1'd. +# +# NOTE: this does NOT update changelogs because the changelog markdown files do not have a reliable machine-readable +# format to automatically handle this. Someday it could though: https://github.com/airbytehq/airbyte/issues/12031 +cmd_bump_version() { + # Take params + local connector_path + local bump_version + connector_path="$1" # Should look like airbyte-integrations/connectors/source-X + bump_version="$2" || bump_version="patch" + + # Set local constants + connector=${connector_path#airbyte-integrations/connectors/} + if [[ "$connector" =~ "source-" ]]; then + connector_type="source" + elif [[ "$connector" =~ "destination-" ]]; then + connector_type="destination" + else + echo "Invalid connector_type from $connector" + exit 1 + fi + definitions_path="./airbyte-config/init/src/main/resources/seed/${connector_type}_definitions.yaml" + dockerfile="$connector_path/Dockerfile" + master_dockerfile="/tmp/master_${connector}_dockerfile" + # This allows getting the contents of a file without checking it out + git --no-pager show "origin/master:$dockerfile" > "$master_dockerfile" + + # Current version always comes from master, this way we can always bump correctly relative to master + # verses a potentially stale local branch + current_version=$(_get_docker_image_version "$master_dockerfile") + local image_name; image_name=$(_get_docker_image_name "$dockerfile") + rm "$master_dockerfile" + + ## Create bumped version + IFS=. read -r major_version minor_version patch_version <<<"${current_version##*-}" + case "$bump_version" in + "major") + ((major_version++)) + minor_version=0 + patch_version=0 + ;; + "minor") + ((minor_version++)) + patch_version=0 + ;; + "patch") + ((patch_version++)) + ;; + *) + echo "Invalid bump_version option: $bump_version. Valid options are major, minor, patch" + exit 1 + esac + + bumped_version="$major_version.$minor_version.$patch_version" + # This image should not already exist, if it does, something weird happened + _error_if_tag_exists "$image_name:$bumped_version" + echo "$connector:$current_version will be bumped to $connector:$bumped_version" + + ## Write new version to files + # 1) Dockerfile + sed -i "s/$current_version/$bumped_version/g" "$dockerfile" + + # 2) Definitions YAML file + definitions_check=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitions_path") + + if [[ (-z "$definitions_check") ]]; then + echo "Could not find $connector in $definitions_path, exiting 1" + exit 1 + fi + + connector_name=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitions_path") + yq e "(.[] | select(.name == \"$connector_name\").dockerImageTag)|=\"$bumped_version\"" -i "$definitions_path" + + # 3) Seed files + ./gradlew :airbyte-config:init:processResources + + echo "Woohoo! Successfully bumped $connector:$current_version to $connector:$bumped_version" +} + cmd_publish() { local path=$1; shift || error "Missing target (root path of integration) $USAGE" [ -d "$path" ] || error "Path must be the root path of the integration" @@ -149,7 +261,7 @@ cmd_publish() { echo "Using environment gcloud" fi - gsutil cp "$tmp_spec_file" gs://io-airbyte-cloud-spec-cache/specs/"$image_name"/"$image_version"/spec.json + gsutil cp "$tmp_spec_file" "gs://io-airbyte-cloud-spec-cache/specs/$image_name/$image_version/spec.json" else echo "Publishing without writing to spec cache." fi @@ -176,7 +288,7 @@ cmd_publish_external() { echo "Using environment gcloud" - gsutil cp "$tmp_spec_file" gs://io-airbyte-cloud-spec-cache/specs/"$image_name"/"$image_version"/spec.json + gsutil cp "$tmp_spec_file" "gs://io-airbyte-cloud-spec-cache/specs/$image_name/$image_version/spec.json" } main() {