gradle.yml

name: Airbyte Connectors & Octavia CI

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}

on:
  #ability to start task manually in Web UI
  workflow_dispatch:
    inputs:
      debug_mode:
        description: "Enable or disable tmate session for debug during helm ac tests"
        type: choice
        default: "false"
        options:
          - "true"
          - "false"
        required: false
  push:
    branches:
      - master
  pull_request:

permissions: write-all

jobs:
  # COMMON TASKS
  ensure-images-exist:
    name: "Ensure all required Docker images exist on Dockerhub"
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3

      - name: Check images exist
        run: ./tools/bin/check_images_exist.sh all
  # The output of this job is used to trigger the following builds.
  changes:
    name: "Detect Modified Files"
    # The filtering action does not deal with well scheduled events so skip to avoid errors.
    # See https://github.com/dorny/paths-filter/issues/100 for more info.
    # This is okay this workflow is only scheduled on master, where we want to build everything
    # so filtering is not required. Use always() in each start block to force the start task.
    if: github.event_name != 'schedule'
    runs-on: ubuntu-latest
    outputs:
      build: ${{ steps.filter.outputs.build }}
      cdk: ${{ steps.filter.outputs.cdk }}
      cli: ${{ steps.filter.outputs.cli }}
      connectors: ${{ steps.filter.outputs.connectors }}
      db: ${{ steps.filter.outputs.db }}
      frontend: ${{ steps.filter.outputs.frontend }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          # Note, the following glob expression within a filters are ORs.
          filters: |
            build:
              - '.github/**'
              - 'buildSrc/**'
              - 'tools/**'
              - '*.gradle'
              - 'deps.toml'
              - 'airbyte-config/**'
            cdk:
              - 'airbyte-cdk/**'
            cli:
              - 'airbyte-api/**'
              - 'octavia-cli/**'
            connectors:
              - 'airbyte-integrations/**'
              - 'airbyte-connector-test-harnesses/acceptance-test-harness/**'
            db:
              - 'airbyte-db/**'

  # Uncomment to debug.
  #  changes-output:
  #    name: "Debug Change Detection Logic"
  #    needs: changes
  #    runs-on: ubuntu-latest
  #    steps:
  #      - uses: actions/checkout@v3
  #      - run: |
  #          echo '${{ toJSON(needs) }}'

  ## BUILDS
  octavia-cli-build:
    needs: changes
    runs-on: ubuntu-latest
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
    name: "Octavia CLI: Build"
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        with:
          ref: ${{ github.head_ref }}

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "17"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Format
        uses: Wandalen/wretry.action@master
        with:
          command: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Ensure no file change
        run: ./tools/bin/check_for_file_changes

      - name: Build
        uses: Wandalen/wretry.action@master
        with:
          command: SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Run integration tests
        uses: Wandalen/wretry.action@master
        with:
          command: ./tools/bin/integration_tests_octavia.sh
          attempt_limit: 3
          attempt_delay: 5000 # in ms

  cdk-build:
    needs: changes
    runs-on: ubuntu-latest
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: needs.changes.outputs.cdk == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
    name: "Airbyte CDK: Build"
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}-cdk

      - uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "17"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Format
        uses: Wandalen/wretry.action@master
        with:
          command: SUB_BUILD=CDK ./gradlew format --scan --info --stacktrace
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Commit Formatting Changes
        uses: stefanzweifel/git-auto-commit-action@v4
        with:
          commit_message: Automated Commit - Formatting Changes
          commit_user_name: Octavia Squidington III
          commit_user_email: octavia-squidington-iii@users.noreply.github.com

      - name: Build
        uses: Wandalen/wretry.action@master
        with:
          command: SUB_BUILD=CDK ./gradlew build --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms


  # Connectors Base
  # In case of self-hosted EC2 errors, remove this block.
  start-connectors-base-build-runner:
    name: "Connectors Base: Start Build EC2 Runner"
    needs:
      - changes
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: |
      needs.changes.outputs.build == 'true' || needs.changes.outputs.connectors == 'true' || needs.changes.outputs.db == 'true' || (always() && github.ref == 'refs/heads/master')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ env.PAT }}

  build-connectors-base:
    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
    needs: start-connectors-base-build-runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner
    name: "Connectors Base: Build"
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        with:
          ref: ${{ github.head_ref }}

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}-connectors

      - uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "17"

      - uses: actions/setup-node@v3
        with:
          node-version: "lts/*"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install Pyenv
        run: python3 -m pip install virtualenv==16.7.9 --user

      - name: Install automake
        run: apt-get install -y automake build-essential libtool libtool-bin autoconf

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Generate Template scaffold
        uses: Wandalen/wretry.action@master
        with:
          command: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      # Verify that the only committed file changes will be those made by the build, formatter or processResources
      - name: Ensure no file change from code formatting
        run: git --no-pager diff && test -z "$(git --no-pager diff)"

      - name: Format
        uses: Wandalen/wretry.action@master
        with:
          command: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Build
        uses: Wandalen/wretry.action@master
        with:
          command: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Process Resources
        uses: Wandalen/wretry.action@master
        with:
          command: SUB_BUILD=CONNECTORS_BASE ./gradlew :airbyte-config:init:processResources --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Commit Changes
        uses: stefanzweifel/git-auto-commit-action@v4
        with:
          commit_message: Automated Change
          commit_user_name: Octavia Squidington III
          commit_user_email: octavia-squidington-iii@users.noreply.github.com

      - name: Publish Connectors Base Test Results
        uses: EnricoMi/publish-unit-test-result-action@v2
        id: connectors-test-results
        if: always()
        with:
          junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
          comment_mode: failures
          json_file: connectors_base_results.json
          json_test_case_results: true
          check_name: "Connectors Base Test Results"

      - name: Setup Google Cloud SDK
        if: always()
        uses: google-github-actions/setup-gcloud@v0
        with:
          service_account_key: ${{ secrets.GKE_TEST_SA_KEY }}
          export_default_credentials: true

      - name: Prep Test Results For GCS
        if: always()
        run: |
          python tools/bin/prep_test_results_for_gcs.py --json connectors_base_results.json  --jobid $GITHUB_JOB --runid $GITHUB_RUN_ID

      - name: Upload Test Results to GCS
        if: always()
        run: |
          gcs_bucket_name="dev-ab-ci-run-results"
          filename=$(echo "${{ fromJSON( steps.connectors-test-results.outputs.json ).check_url }}" | sed 's@.*/@@')
          echo "$filename"
          gsutil -h "Cache-Control:public" cp connectors_base_results.jsonl "gs://$gcs_bucket_name/oss/$filename.jsonl"

      - name: Generate Test Report
        uses: dorny/test-reporter@v1
        if: always()
        with:
          name: Connectors Base Test Report
          # Specify top-level and second-level modules. Note there cannot be a space between the comma.
          path: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
          reporter: java-junit
          fail-on-error: "false"

  # In case of self-hosted EC2 errors, remove this block.
  stop-connectors-base-build-runner:
    name: "Connectors Base: Stop Build EC2 Runner"
    timeout-minutes: 10
    needs:
      - start-connectors-base-build-runner # required to get output from the start-runner job
      - build-connectors-base # required to wait when the main job is done
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-connectors-base-build-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Stop EC2 runner
        uses: supertopher/ec2-github-runner@base64v1.0.10
        with:
          mode: stop
          github-token: ${{ env.PAT }}
          label: ${{ needs.start-connectors-base-build-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }}

  notify-failure-slack-channel:
    name: "Notify Slack Channel on Build Failures"
    runs-on: ubuntu-latest
    needs:
      - build-connectors-base
      - octavia-cli-build
    if: ${{ failure() && github.ref == 'refs/heads/master' }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Match GitHub User to Slack User
        id: match-github-to-slack-user
        uses: ./.github/actions/match-github-to-slack-user
        env:
          AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Publish to OSS Build Failure Slack Channel
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
            {\"type\":\"divider\"}]}

  notify-failure-slack-channel-fixed-broken-build:
    name: "Notify Slack Channel on Build Fixes"
    runs-on: ubuntu-latest
    needs:
      - build-connectors-base
      - octavia-cli-build
    if: success()
    steps:
      - name: Get Previous Workflow Status
        uses: Mercymeilya/last-workflow-status@v0.3
        id: last_status
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
      # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
      - name: Publish Build Fixed Message to OSS Build Failure Slack Channel
        if: ${{ steps.last_status.outputs.last_status == 'failure' }}
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
            {\"type\":\"divider\"}]}