179 lines (159 loc) · 7.37 KB
/
simulator-nightly.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
name: Simulator Tests
concurrency:
group: ${{ github.workflow }}
on:
schedule:
- cron: '0 9 * * *' # equivalent to 1am PST
workflow_dispatch:
inputs:
sui_ref:
description: "Branch / commit to test"
type: string
required: true
default: main
test_num:
description: "MSIM_TEST_NUM (test iterations)"
type: string
required: false
default: "30"
env:
SUI_REF: "${{ github.event.inputs.sui_ref || 'main' }}"
TEST_NUM: "${{ github.event.inputs.test_num || '30' }}"
jobs:
simtest:
# Allow running the job for up to 6 hours, maximum for GitHub Actions.
timeout-minutes: 360
permissions:
# The "id-token: write" permission is required or Machine ID will not be
# able to authenticate with the cluster.
id-token: write
contents: read
runs-on: ubuntu-latest
steps:
- name: Install Teleport
uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # pin@v1.0.6
with:
version: 11.3.1
- name: Authorize against Teleport
id: auth
uses: teleport-actions/auth@9091dad16a564f3c5b9c2ec520b234a4872b6879 # pin@v1
with:
# Specify the publically accessible address of your Teleport proxy.
proxy: proxy.mysten-int.com:443
# Specify the name of the join token for your bot.
token: sui-simtest-token
# Specify the length of time that the generated credentials should be
# valid for. This is optional and defaults to "1h"
certificate-ttl: 2h
# Cargo clean and git restore on any left over files from git checkout, and deletes all remote tracking branches
- name: Environment clean
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && rm -rf ~/sui"
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/ && git clone git@github.com:MystenLabs/sui.git"
# Deleting files in tmpfs that usually fill up pretty quickly after each run. Cargo clean to free up space as well.
- name: Tmpfs and cargo clean
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "sudo rm -rf /tmp/*"
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && cargo clean"
# Checkout out the latest sui repo
- name: Checkout sui repo
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && git fetch origin && git checkout ${{ env.SUI_REF }}"
# Setting up cargo and simtest
- name: Install simtest
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && ./scripts/simtest/install.sh"
# Run simulator tests
- name: Run simtest
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 120 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && RUSTUP_MAX_RETRIES=10 CARGO_TERM_COLOR=always CARGO_INCREMENTAL=0 CARGO_NET_RETRY=10 RUST_BACKTRACE=short RUST_LOG=off NUM_CPUS=24 TEST_NUM=${{ env.TEST_NUM }} ./scripts/simtest/simtest-run.sh"
notify:
name: Notify
needs: [simtest]
runs-on: ubuntu-latest
if: github.event_name == 'schedule' && failure()
steps:
- uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # Pin v4.1.1
- name: Checkout sui repo main branch
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
- name: Get sui commit
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
export sui_sha=$(git rev-parse HEAD)
echo "sui_sha=${sui_sha}" >> $GITHUB_ENV
- name: Get link to logs
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh_job_link=$(gh api -X GET 'repos/MystenLabs/sui/actions/runs/${{ github.run_id }}/jobs' --jq '.jobs.[0].html_url')
echo "gh_job_link=${gh_job_link}" >> $GITHUB_ENV
- name: Get current oncall
run: |
export current_oncall=$(curl -s --request GET \
--url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \
--header 'Accept: application/json' \
--header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \
--header 'Content-Type: application/json' \
| jq '.oncalls[].user.summary' | tr -d '"')
echo "current_oncall=$(echo ${current_oncall})" >> $GITHUB_ENV
export oncall_name=$(curl -s --request GET \
--url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \
--header 'Accept: application/json' \
--header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \
--header 'Content-Type: application/json' \
| jq '.oncalls[].escalation_policy.summary' | tr -d '"')
echo "oncall_name=$(echo ${oncall_name})" >> $GITHUB_ENV
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # pin v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Get slack id for the oncall
run: |
export slack_id=$(aws s3 cp s3://mysten-employees-dir/employees.json - | jq --arg ONCALL "${{ env.current_oncall }}" '.[] | if .name == $ONCALL then .slack_id else empty end')
echo "slack_id=$(echo ${slack_id} | tr -d '"')" >> $GITHUB_ENV
- name: Post to slack
uses: slackapi/slack-github-action@v1.24.0 # pin@v1.21.0
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SUI_SHA: ${{ env.sui_sha }}
GH_JOB_LINK: ${{ env.gh_job_link }}
SLACK_ID: ${{ env.slack_id }}
ONCALL_NAME: ${{ env.oncall_name }}
with:
channel-id: 'simtest-nightly'
payload: |
{
"text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Sui commit: <https://github.com/MystenLabs/sui/commit/${{ env.SUI_SHA }}|${{ env.SUI_SHA }}> \nRun: <${{ env.GH_JOB_LINK }}|${{ github.run_id }}>"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "<@${{ env.SLACK_ID }}>, current `${{ env.ONCALL_NAME }}` oncall, please debug failures: `tsh ssh ubuntu@simtest-01` and look in the `/home/ubuntu/simtest_logs/{date}` folder for test results"
}
}
]
}