-
Notifications
You must be signed in to change notification settings - Fork 3.8k
217 lines (205 loc) · 8.94 KB
/
gradle.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
name: Airbyte CI - Repository Health Check
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
on:
workflow_dispatch:
push:
branches:
- master
pull_request:
types:
- opened
- reopened
- synchronize
jobs:
# In case of self-hosted EC2 errors, remove this block.
start-check-runner:
name: Start EC2 Runner
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
# Use a beefier instance type than the default c5.2xlarge, but with the same per-core cost.
# When gradle runs on this instance, it will use up all the available cores anyway.
# There should be little to no difference in total cost, however the job latency will be improved.
# At the time of this writing, the latency doesn't improve much beyond this instance size (approx 5 minutes).
# This is largely thanks to the gradle cache.
ec2-instance-type: "c5.4xlarge"
github-token: ${{ env.PAT }}
run-check:
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
needs: start-check-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner
name: Gradle Check
timeout-minutes: 30
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
# IMPORTANT! This is nessesary to make sure that a status is reported on the PR
# even if the workflow is skipped. If we used github actions filters, the workflow
# would not be reported as skipped, but instead would be forever pending.
#
# I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
#
# Also it gets worse
#
# IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
# MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
- name: Get changed files
uses: tj-actions/changed-files@v39
id: changes
with:
files_yaml: |
gradlecheck:
- '**/*'
- '!**/*.md'
- uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
- name: Install Pip
if: steps.changes.outputs.gradlecheck_any_changed == 'true'
run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3
- name: Install Pyenv
if: steps.changes.outputs.gradlecheck_any_changed == 'true'
run: python3 -m pip install virtualenv --user
- name: Run Gradle Check
if: steps.changes.outputs.gradlecheck_any_changed == 'true'
uses: burrunan/gradle-cache-action@v1
env:
CI: true
with:
read-only: ${{ github.ref != 'refs/heads/master' }}
# TODO: be able to remove the skipSlowTests property
# TODO: remove the format task ASAP
# TODO: roll the CDK tasks into check
arguments: --scan --no-daemon --no-watch-fs format check -DskipSlowTests=true
# In case of self-hosted EC2 errors, remove this block.
stop-check-runner:
name: Stop EC2 Runner
timeout-minutes: 10
needs:
- start-check-runner # required to get output from the start-runner job
- run-check # required to wait when the main job is done
runs-on: ubuntu-latest
# Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
# Thus, we check for skipped here.
if: ${{ always() && needs.start-check-runner.result != 'skipped'}}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
- name: Stop EC2 runner
uses: supertopher/ec2-github-runner@base64v1.0.10
with:
mode: stop
github-token: ${{ env.PAT }}
label: ${{ needs.start-check-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-check-runner.outputs.ec2-instance-id }}
set-instatus-incident-on-failure:
name: Create Instatus Incident on Failure
runs-on: ubuntu-latest
needs:
- run-check
if: ${{ failure() && github.ref == 'refs/heads/master' }}
steps:
- name: Call Instatus Webhook
uses: joelwmale/webhook-action@master
with:
url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
body: '{ "trigger": "down", "status": "HASISSUES" }'
set-instatus-incident-on-success:
name: Create Instatus Incident on Success
runs-on: ubuntu-latest
needs:
- run-check
if: ${{ success() && github.ref == 'refs/heads/master' }}
steps:
- name: Call Instatus Webhook
uses: joelwmale/webhook-action@master
with:
url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
body: '{ "trigger": "up" }'
notify-failure-slack-channel:
name: "Notify Slack Channel on Build Failures"
runs-on: ubuntu-latest
needs:
- run-check
if: ${{ failure() && github.ref == 'refs/heads/master' }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Match GitHub User to Slack User
id: match-github-to-slack-user
uses: ./.github/actions/match-github-to-slack-user
env:
AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Publish to OSS Build Failure Slack Channel
uses: abinoda/slack-action@master
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
with:
args: >-
{\"channel\":\"C03BEADRPNY\", \"blocks\":[
{\"type\":\"divider\"},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
{\"type\":\"divider\"}]}
notify-failure-slack-channel-fixed-broken-build:
name: "Notify Slack Channel on Build Fixes"
runs-on: ubuntu-latest
needs:
- run-check
if: success()
steps:
- name: Get Previous Workflow Status
uses: Mercymeilya/last-workflow-status@v0.3
id: last_status
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
# To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
- name: Publish Build Fixed Message to OSS Build Failure Slack Channel
if: ${{ steps.last_status.outputs.last_status == 'failure' }}
uses: abinoda/slack-action@master
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
with:
args: >-
{\"channel\":\"C03BEADRPNY\", \"blocks\":[
{\"type\":\"divider\"},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
{\"type\":\"divider\"}]}