feat(cron): get and persist latest declarative manifest docker image … #5610
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The goal of this build is to make sure that OSS contributors can build the project and run the tests | |
| # so that they can develop locally. It is NOT a release verification. As such, we just run build and | |
| # unit test. No additional acceptance test, etc. | |
| name: Airbyte Platform OSS Developer Build | |
| env: | |
| S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| on: | |
| #ability to start task manually in Web UI | |
| workflow_dispatch: | |
| inputs: | |
| debug_mode: | |
| description: "Enable or disable tmate session for debug during helm ac tests" | |
| type: choice | |
| default: "false" | |
| options: | |
| - "true" | |
| - "false" | |
| required: false | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| permissions: write-all | |
| jobs: | |
| ensure-images-exist: | |
| name: "Ensure all required Docker images exist on Dockerhub" | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check images exist | |
| run: ./tools/bin/check_images_exist.sh all | |
| # The output of this job is used to trigger the following builds. | |
| changes: | |
| name: "Detect Modified Files" | |
| # The filtering action does not deal with well scheduled events so skip to avoid errors. | |
| # See https://github.com/dorny/paths-filter/issues/100 for more info. | |
| # This is okay this workflow is only scheduled on main, where we want to build everything | |
| # so filtering is not required. Use always() in each start block to force the start task. | |
| if: github.event_name != 'schedule' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend: ${{ steps.filter.outputs.backend }} | |
| build: ${{ steps.filter.outputs.build }} | |
| cli: ${{ steps.filter.outputs.cli }} | |
| db: ${{ steps.filter.outputs.db }} | |
| frontend: ${{ steps.filter.outputs.frontend }} | |
| frontendServers: ${{ steps.changes.outputs.frontendServers }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| # Note, the following glob expression within a filters are ORs. | |
| filters: | | |
| backend: | |
| - 'airbyte-!(cdk|integrations|webapp)/**' | |
| - 'airbyte-integrations/connectors/(destination-jdbc|destination-postgres|source-jdbc|source-postgres)/**' | |
| - 'airbyte-config/init/src/main/resources/seed/(source|destination)_definitions.yaml' | |
| - 'docker-compose*.yaml' | |
| - '(charts|kube)/**' | |
| build: | |
| - '.github/**' | |
| - 'buildSrc/**' | |
| - 'tools/**' | |
| - '*.gradle' | |
| - 'deps.toml' | |
| cli: | |
| - 'airbyte-api/**' | |
| db: | |
| - 'airbyte-db/**' | |
| frontend: | |
| - 'airbyte-api/src/main/openapi/config.yaml' | |
| - 'oss/airbyte-api/src/main/openapi/cloud-config.yaml' | |
| - 'airbyte-connector-builder-resources/CDK_VERSION' | |
| - 'airbyte-connector-builder-server/src/main/openapi/openapi.yaml' | |
| - 'airbyte-webapp/**' | |
| frontendServers: | |
| - 'airbyte-(server|data|connector-builder-server|featureflag|commons|commons-!(worker|temporal*))/**' | |
| # Uncomment to debug. | |
| # changes-output: | |
| # name: "Debug Change Detection Logic" | |
| # needs: changes | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - run: | | |
| # echo '${{ toJSON(needs) }}' | |
| platform-build: | |
| name: "Platform: Build" | |
| # At the time of writing, using the beefier AWS runners made this step take 7 minutes instead of 10 minutes | |
| # on the stock runners. Since no developer iteration loop depends on this, we use the stock gh runners. If | |
| # we need to bring back the AWS runners, checkout 98f3e85a9ea2634298653642b18ac38fd1b85929 as a reference. | |
| runs-on: ubuntu-latest | |
| needs: | |
| - changes | |
| # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on master. | |
| if: | | |
| needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main' | |
| || (always() && needs.changes.outputs.frontendServers == 'true') | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Cache Build Artifacts | |
| uses: ./.github/actions/cache-build-artifacts | |
| with: | |
| cache-key: ${{ secrets.CACHE_VERSION }} | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: "zulu" | |
| java-version: "21" | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: "lts/*" | |
| - name: Set up CI Gradle Properties | |
| run: | | |
| mkdir -p ~/.gradle/ | |
| cat > ~/.gradle/gradle.properties <<EOF | |
| org.gradle.jvmargs=-Xmx8g -Xss4m \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED | |
| org.gradle.workers.max=8 | |
| org.gradle.vfs.watch=false | |
| EOF | |
| - name: Format | |
| uses: Wandalen/wretry.action@v1.0.36 | |
| with: | |
| command: ./gradlew format --scan --info --stacktrace | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - name: Ensure no file change | |
| run: git --no-pager diff && test -z "$(git --no-pager diff)" | |
| - name: Build | |
| uses: Wandalen/wretry.action@v1.0.36 | |
| with: | |
| # todo (cgardens): temporarily skipping airbyte-commons-worker because one of the tests uses too much memory. we are | |
| # adjusting the test to use a more reasonable amount of memory. https://github.com/airbytehq/oncall/issues/4415 | |
| # command: ./gradlew build javadoc --scan | |
| command: ./gradlew build -x :airbyte-commons-worker:test javadoc --scan | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - name: Check documentation | |
| if: success() && github.ref == 'refs/heads/main' | |
| run: ./tools/site/link_checker.sh check_docs | |
| # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. | |
| # - name: Get Docker Space | |
| # run: docker run --rm busybox df -h | |
| notify-failure-slack-channel: | |
| name: "Notify Slack Channel on Build Failures" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - platform-build | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Match GitHub User to Slack User | |
| id: match-github-to-slack-user | |
| uses: ./.github/actions/match-github-to-slack-user | |
| env: | |
| AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} | |
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Publish to OSS Build Failure Slack Channel | |
| uses: abinoda/slack-action@master | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| args: >- | |
| {\"channel\":\"C03BEADRPNY\", \"blocks\":[ | |
| {\"type\":\"divider\"}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS main failed! :bangbang: \n\n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}}, | |
| {\"type\":\"divider\"}]} | |
| notify-failure-slack-channel-fixed-broken-build: | |
| name: "Notify Slack Channel on Build Fixes" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - platform-build | |
| if: ${{ success() && github.ref == 'refs/heads/main' }} | |
| steps: | |
| - name: Get Previous Workflow Status | |
| uses: Mercymeilya/last-workflow-status@v0.3 | |
| id: last_status | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. | |
| - name: Publish Build Fixed Message to OSS Build Failure Slack Channel | |
| if: ${{ steps.last_status.outputs.last_status == 'failure' }} | |
| uses: abinoda/slack-action@master | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| args: >- | |
| {\"channel\":\"C03BEADRPNY\", \"blocks\":[ | |
| {\"type\":\"divider\"}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Main Fixed! :white_check_mark: \n\n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}}, | |
| {\"type\":\"divider\"}]} |