Merge branch 'master' into george/fb-marketing

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.36.5-alpha
+current_version = 0.36.6-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.36.5-alpha
+VERSION=0.36.6-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/workflows/build-connector-command.yml

@@ -0,0 +1,260 @@
+name: Bump, Build, Test Connectors [EXPERIMENTAL]
+on:
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
+        required: false
+        default: "airbytehq/airbyte"
+      gitref:
+        description: "The git ref to check out from the specified repository."
+        required: false
+        default: master
+      connector:
+        description: "Airbyte Connector"
+        required: true
+      bump-version:
+        description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually"
+        required: false
+        default: "false"
+      run-tests:
+        description: "Should run tests"
+        required: false
+        default: "true"
+      comment-id:
+        description: "The comment-id of the slash command. Used to update the comment with the status."
+        required: false
+
+jobs:
+  find_valid_pat:
+    name: "Find a PAT with room for actions"
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      pat: ${{ steps.variables.outputs.pat }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        id: variables
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
+            ${{ secrets.SUPERTOPHER_PAT }} \
+            ${{ secrets.DAVINCHIA_PAT }}
+  ## Gradle Build
+  # In case of self-hosted EC2 errors, remove this block.
+  start-bump-build-test-connector-runner:
+    name: Start Build EC2 Runner
+    runs-on: ubuntu-latest
+    needs: find_valid_pat
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ github.event.inputs.repo }}
+          ref: ${{ github.event.inputs.gitref }}
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+          # 80 gb disk
+          ec2-image-id: ami-0d648081937c75a73
+  bump-build-test-connector:
+    name: Bump, Build, Test Connector
+    needs: start-bump-build-test-connector-runner
+    runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
+    environment: more-secrets
+    steps:
+      ############################
+      ##         SET UP         ##
+      ############################
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
+          export_default_credentials: true
+      - name: Search for valid connector name format
+        id: regex
+        uses: AsasInnab/regex-action@v1
+        with:
+          regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$"
+          regex_flags: "i" # required to be set for this plugin
+          search_string: ${{ github.event.inputs.connector }}
+      - name: Validate input workflow format
+        if: steps.regex.outputs.first_match != github.event.inputs.connector
+        run: echo "The connector provided has an invalid format!" && exit 1
+      - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ github.event.inputs.repo }}
+          ref: ${{ github.event.inputs.gitref }}
+          token: ${{ secrets.OCTAVIA_PAT }}
+      - name: Install Java
+        uses: actions/setup-java@v1
+        with:
+          java-version: "17"
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+      - name: Install Pyenv and Tox
+        run: |
+          python3 -m pip install --quiet virtualenv==16.7.9 --user
+          python3 -m virtualenv venv
+          source venv/bin/activate
+          pip install --quiet tox==3.24.4
+      - name: Install yq
+        if: github.event.inputs.bump-version != 'false' && success()
+        run: |
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+          sudo add-apt-repository ppa:rmescandon/yq
+          sudo apt update
+          sudo apt install yq -y
+      - name: Test and install CI scripts
+        # all CI python packages have the prefix "ci_"
+        run: |
+          source venv/bin/activate
+          tox -r -c ./tools/tox_ci.ini
+          pip install --quiet -e ./tools/ci_*
+      - name: Get Credentials for ${{ github.event.inputs.connector }}
+        run: |
+          source venv/bin/activate
+          ci_credentials ${{ github.event.inputs.connector }}
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+      # TODO: seems like this should run in post-merge workflow
+      #      - name: Prepare Sentry
+      #        if: startsWith(github.event.inputs.connector, 'connectors')
+      #        run: |
+      #          curl -sL https://sentry.io/get-cli/ | bash
+      #      - name: Create Sentry Release
+      #        if: startsWith(github.event.inputs.connector, 'connectors')
+      #        run: |
+      #          sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing
+      #        env:
+      #          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
+      #          SENTRY_ORG: airbyte-5j
+      #          SENTRY_PROJECT: airbyte-connectors
+      ############################
+      ##          BUMP          ##
+      ############################
+      - name: Bump Connector Version
+        if: github.event.inputs.bump-version != 'false' && success()
+        run: ./tools/integrations/manage.sh bump_version airbyte-integrations/${{ github.event.inputs.connector }}
+      - name: Commit and Push Version Bump
+        if: github.event.inputs.bump-version != 'false' && success()
+        run: |
+          git config user.name 'Octavia Squidington III'
+          git config user.email 'octavia-squidington-iii@users.noreply.github.com'
+          git add -u
+          git commit -m "bump-version ${{github.event.inputs.connector}}"
+          git push origin ${{ github.event.inputs.gitref }}
+      - name: Add Version Bump Success Comment
+        if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :rocket: Bumped version for ${{github.event.inputs.connector}}
+      - name: Add Version Bump Failure Comment
+        if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && !success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :x: Couldn't bump version for ${{github.event.inputs.connector}}
+      ############################
+      ##     BUILD AND TEST     ##
+      ############################
+      - name: Build ${{ github.event.inputs.connector }}
+        run: ./tools/integrations/manage.sh build_experiment airbyte-integrations/${{ github.event.inputs.connector }}
+        id: build
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
+          TZ: UTC
+  #      - name: Test ${{ github.event.inputs.connector }}
+  #        if: github.event.inputs.run-tests == 'true'
+  #        run: ./tools/integrations/manage.sh test airbyte-integrations/${{ github.event.inputs.connector }}
+  #      - name: Finalize Sentry release
+  #        if: startsWith(github.event.inputs.connector, 'connectors')
+  #        run: |
+  #          sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}"
+  #        env:
+  #          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
+  #          SENTRY_ORG: airbyte-5j
+  #          SENTRY_PROJECT: airbyte-connectors
+  #      - name: Build and Test Success Comment
+  #        if: github.event.inputs.comment-id && success()
+  #        uses: peter-evans/create-or-update-comment@v1
+  #        with:
+  #          comment-id: ${{ github.event.inputs.comment-id }}
+  #          body: |
+  #            > :rocket: Successfully built and tested ${{github.event.inputs.connector}}
+  #      - name: Build and Test Failure Comment
+  #        if: github.event.inputs.comment-id && !success()
+  #        uses: peter-evans/create-or-update-comment@v1
+  #        with:
+  #          comment-id: ${{ github.event.inputs.comment-id }}
+  #          body: |
+  #            > :x: Failed to build and test ${{github.event.inputs.connector}}
+  #      - name: Slack Notification - Failure
+  #        if: failure()
+  #        uses: rtCamp/action-slack-notify@master
+  #        env:
+  #          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+  #          SLACK_USERNAME: Buildozer
+  #          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
+  #          SLACK_COLOR: DC143C
+  #          SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}"
+  #          SLACK_FOOTER: ""
+  #      - name: Add Final Success Comment
+  #        if: github.event.inputs.comment-id && success()
+  #        uses: peter-evans/create-or-update-comment@v1
+  #        with:
+  #          comment-id: ${{ github.event.inputs.comment-id }}
+  #          body: |
+  #            > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+  #      - name: Set publish label
+  #        if: success()
+  #        run: |
+  #          echo "set some label on PR"
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-bump-build-test-connector-runner:
+    name: Stop Build EC2 Runner
+    needs:
+      - start-bump-build-test-connector-runner # required to get output from the start-runner job
+      - bump-build-test-connector # required to wait when the main job is done
+      - find_valid_pat
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: supertopher/ec2-github-runner@base64v1.0.10
+        with:
+          mode: stop
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+          label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }}