Skip to content

Commit 2f9c021

Browse files
galarghBigLep
galargh
and
BigLep
authored
feat: set up very expensive tests to run in CI (#12939)
* ci: run expensive tests in the CI * ci: make the test workflow reusable * ci: run very expensive tests on label addition and on schedule * ci: limit the number of tests executed by the very expensive test runner * ci: fix the test workflow setup * ci: do not cache dependencies when running very expensive tests * ci: do not wait for very expensive tests to finish to remove the label * ci: fix the label reference * ci: ensure the very expensive tests get executed * ci: do cache very expensive tests after all * ci: increase buffer sizes * chore: apply suggestions from code review Co-authored-by: Steve Loeppky <biglep@filoz.org> * ci: update very expensive test trigger and add memory monitoring * ci: run very-expensive-tests on network optimized runners * ci: monitor free memory only on debug reruns * wip * feat: do not create new issues if one already exists --------- Co-authored-by: Steve Loeppky <biglep@filoz.org>
1 parent 090432a commit 2f9c021

File tree

3 files changed

+296
-1
lines changed

3 files changed

+296
-1
lines changed

‎.github/workflows/reusable-test.yml‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
on:
2+
workflow_call:
3+
inputs:
4+
run_very_expensive_tests:
5+
description: 'Run very expensive tests'
6+
required: false
7+
default: false
8+
type: boolean
9+
10+
defaults:
11+
run:
12+
shell: bash
13+
14+
permissions:
15+
contents: read
16+
17+
jobs:
18+
discover:
19+
name: Discover Test Groups
20+
runs-on: ubuntu-latest
21+
outputs:
22+
test_group_execution_contexts: ${{ steps.list_test_group_execution_contexts.outputs.test_group_execution_contexts }}
23+
steps:
24+
- uses: actions/checkout@v4
25+
with:
26+
submodules: 'recursive'
27+
fetch-depth: 0
28+
- id: list_test_group_execution_contexts
29+
env:
30+
VERY_EXPENSIVE_TESTS_RUN: ${{ inputs.run_very_expensive_tests }}
31+
run: |
32+
echo "test_group_execution_contexts<<EOF" >> $GITHUB_OUTPUT
33+
go run ./cmd/ci/main.go --json list-test-group-execution-contexts --very-expensive-tests-run=$VERY_EXPENSIVE_TESTS_RUN | jq -r '.msg' | tee -a $GITHUB_OUTPUT
34+
echo "EOF" >> $GITHUB_OUTPUT
35+
cache:
36+
name: Cache Dependencies
37+
runs-on: ${{ matrix.runner }}
38+
strategy:
39+
matrix:
40+
# We need to cache for each architecture we support: x86_64 and arm64
41+
runner: [ubuntu-latest, ubuntu-24.04-arm]
42+
steps:
43+
- uses: actions/checkout@v4
44+
with:
45+
submodules: 'recursive'
46+
fetch-depth: 0
47+
- id: fetch_params
48+
env:
49+
CACHE_KEY: fetch-params-${{ hashFiles('./extern/filecoin-ffi/parameters.json') }}
50+
CACHE_PATH: |
51+
/var/tmp/filecoin-proof-parameters/
52+
run: |
53+
echo -e "key=$CACHE_KEY" | tee -a $GITHUB_OUTPUT
54+
echo -e "path<<EOF\n$CACHE_PATH\nEOF" | tee -a $GITHUB_OUTPUT
55+
- id: make_deps
56+
env:
57+
CACHE_KEY: ${{ runner.os }}-${{ runner.arch }}-make-deps-${{ hashFiles('./.git/modules/extern/filecoin-ffi/HEAD') }}-p
58+
CACHE_PATH: |
59+
./extern/filecoin-ffi/filcrypto.h
60+
./extern/filecoin-ffi/libfilcrypto.a
61+
./extern/filecoin-ffi/filcrypto.pc
62+
run: |
63+
echo -e "key=$CACHE_KEY" | tee -a $GITHUB_OUTPUT
64+
echo -e "path<<EOF\n$CACHE_PATH\nEOF" | tee -a $GITHUB_OUTPUT
65+
- id: restore_fetch_params
66+
uses: actions/cache/restore@v4
67+
with:
68+
key: ${{ steps.fetch_params.outputs.key }}
69+
path: ${{ steps.fetch_params.outputs.path }}
70+
lookup-only: true
71+
- id: restore_make_deps
72+
uses: actions/cache/restore@v4
73+
with:
74+
key: ${{ steps.make_deps.outputs.key }}
75+
path: ${{ steps.make_deps.outputs.path }}
76+
lookup-only: true
77+
- if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
78+
uses: ./.github/actions/install-system-dependencies
79+
- if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
80+
uses: ./.github/actions/install-go
81+
- if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
82+
env:
83+
GITHUB_TOKEN: ${{ github.token }}
84+
run: FFI_PORTABLE=1 make deps
85+
- if: steps.restore_fetch_params.outputs.cache-hit != 'true'
86+
run: make lotus
87+
- if: steps.restore_fetch_params.outputs.cache-hit != 'true'
88+
run: ./lotus fetch-params 2048
89+
- if: steps.restore_fetch_params.outputs.cache-hit != 'true'
90+
uses: actions/cache/save@v4
91+
with:
92+
key: ${{ steps.fetch_params.outputs.key }}
93+
path: ${{ steps.fetch_params.outputs.path }}
94+
- if: steps.restore_make_deps.outputs.cache-hit != 'true'
95+
uses: actions/cache/save@v4
96+
with:
97+
key: ${{ steps.make_deps.outputs.key }}
98+
path: ${{ steps.make_deps.outputs.path }}
99+
- env:
100+
fetch_params_key: ${{ steps.fetch_params.outputs.key }}
101+
fetch_params_path: ${{ steps.fetch_params.outputs.path }}
102+
make_deps_key: ${{ steps.make_deps.outputs.key }}
103+
make_deps_path: ${{ steps.make_deps.outputs.path }}
104+
file: jobs.cache.${{ runner.os }}.${{ runner.arch }}.outputs.json
105+
run: |
106+
jq -n '{
107+
"fetch_params_key": env.fetch_params_key,
108+
"fetch_params_path": env.fetch_params_path,
109+
"make_deps_key": env.make_deps_key,
110+
"make_deps_path": env.make_deps_path
111+
}' | tee -a "$file"
112+
- uses: actions/upload-artifact@v4
113+
with:
114+
name: jobs.cache.${{ runner.os }}.${{ runner.arch }}.outputs
115+
path: jobs.cache.${{ runner.os }}.${{ runner.arch }}.outputs.json
116+
test:
117+
needs: [discover, cache]
118+
name: Test (${{ matrix.name }}) ${{ toJson(matrix.runner) }}
119+
runs-on: ${{ matrix.runner }}
120+
strategy:
121+
fail-fast: false
122+
matrix:
123+
include: ${{ fromJson(needs.discover.outputs.test_group_execution_contexts) }}
124+
steps:
125+
- uses: actions/checkout@v4
126+
with:
127+
submodules: 'recursive'
128+
fetch-depth: 0
129+
- uses: ./.github/actions/install-system-dependencies
130+
- uses: ./.github/actions/install-go
131+
- id: group
132+
run: |
133+
echo "metadata<<EOF" >> $GITHUB_OUTPUT
134+
go run ./cmd/ci/main.go --json get-test-group-metadata --name "${{ matrix.name }}" | jq -r '.msg' | tee -a $GITHUB_OUTPUT
135+
echo "EOF" >> $GITHUB_OUTPUT
136+
- name: Install gotestsum
137+
run: go install gotest.tools/gotestsum@latest
138+
- id: artifact
139+
uses: actions/download-artifact@v4
140+
with:
141+
name: jobs.cache.${{ runner.os }}.${{ runner.arch }}.outputs
142+
- id: cache
143+
env:
144+
file: jobs.cache.${{ runner.os }}.${{ runner.arch }}.outputs.json
145+
run: |
146+
echo "make_deps_key=$(jq -r .make_deps_key "$file")" | tee -a $GITHUB_OUTPUT
147+
echo "make_deps_path<<EOF" | tee -a $GITHUB_OUTPUT
148+
jq -r .make_deps_path "$file" | tee -a $GITHUB_OUTPUT
149+
echo "EOF" | tee -a $GITHUB_OUTPUT
150+
151+
echo "fetch_params_key=$(jq -r .fetch_params_key "$file")" | tee -a $GITHUB_OUTPUT
152+
echo "fetch_params_path<<EOF" | tee -a $GITHUB_OUTPUT
153+
jq -r .fetch_params_path "$file" | tee -a $GITHUB_OUTPUT
154+
echo "EOF" | tee -a $GITHUB_OUTPUT
155+
156+
rm "$file"
157+
- name: Restore cached make deps outputs
158+
uses: actions/cache/restore@v4
159+
with:
160+
key: ${{ steps.cache.outputs.make_deps_key }}
161+
path: ${{ steps.cache.outputs.make_deps_path }}
162+
fail-on-cache-miss: true
163+
- if: ${{ fromJson(steps.group.outputs.metadata).needs_parameters }}
164+
name: Restore cached fetch params outputs
165+
uses: actions/cache/restore@v4
166+
with:
167+
key: ${{ steps.cache.outputs.fetch_params_key }}
168+
path: ${{ steps.cache.outputs.fetch_params_path }}
169+
fail-on-cache-miss: true
170+
# https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes
171+
- name: Increase UDP buffer sizes
172+
run: |
173+
sudo sysctl -w net.core.rmem_max=7500000
174+
sudo sysctl -w net.core.wmem_max=7500000
175+
# TODO: Install statediff (used to be used for conformance)
176+
- name: Create temporary directory for reports
177+
id: reports
178+
run: mktemp -d | xargs -0 -I{} echo "path={}" | tee -a $GITHUB_OUTPUT
179+
# TODO: Track coverage (used to be tracked for conformance)
180+
- name: Run tests
181+
id: tests
182+
env:
183+
NAME: ${{ matrix.name }}
184+
LOTUS_SRC_DIR: ${{ github.workspace }}
185+
REPORTS_PATH: ${{ steps.reports.outputs.path }}
186+
SKIP_CONFORMANCE: ${{ fromJson(steps.group.outputs.metadata).skip_conformance && '1' || '0' }}
187+
TEST_RUSTPROOFS_LOGS: ${{ fromJson(steps.group.outputs.metadata).test_rust_proofs_logs && '1' || '0' }}
188+
LOTUS_RUN_EXPENSIVE_TESTS: 1
189+
LOTUS_RUN_VERY_EXPENSIVE_TESTS: ${{ inputs.run_very_expensive_tests && '1' || '0' }}
190+
FORMAT: ${{ fromJson(steps.group.outputs.metadata).format || 'standard-verbose' }}
191+
PACKAGES: ${{ join(fromJson(steps.group.outputs.metadata).packages, ' ') }}
192+
TIMEOUT: ${{ inputs.run_very_expensive_tests && '60m' || '10m' }}
193+
MONITOR_FREE_MEMORY: ${{ runner.debug }}
194+
run: |
195+
if [[ "$MONITOR_FREE_MEMORY" == "1" ]]; then
196+
while true; do
197+
free -m
198+
sleep 5
199+
done &
200+
fi
201+
gotestsum \
202+
--format "$FORMAT" \
203+
--junitfile "$REPORTS_PATH/$NAME.xml" \
204+
--jsonfile "$REPORTS_PATH/$NAME.json" \
205+
--packages="$PACKAGES" \
206+
-- -timeout "$TIMEOUT" ${{ fromJson(steps.group.outputs.metadata).go_test_flags || '' }}
207+
- name: Modify junit.xml for BuildPulse
208+
env:
209+
NAME: ${{ matrix.name }}
210+
REPORTS_PATH: ${{ steps.reports.outputs.path }}
211+
PACKAGES: ${{ join(fromJson(steps.group.outputs.metadata).packages, ' ') }}
212+
if: (!cancelled())
213+
run: |
214+
# Modify test suite name and classname attributes in JUnit XML for better grouping
215+
# in BuildPulse. itests are run with go test ./itests/file_test.go and therefore Go
216+
# assigns the name and classname attributes to "command-line-arguments". Others get the
217+
# package name for both.
218+
if [[ "${{ matrix.name }}" == itest-* ]]; then
219+
PACKAGE_NAME=$(basename "$PACKAGES" .go)
220+
sed -i 's/ name="command-line-arguments"/ name="itests"/g' "$REPORTS_PATH/$NAME.xml"
221+
sed -i 's/classname="command-line-arguments"/classname="'"$PACKAGE_NAME"'"/g' "$REPORTS_PATH/$NAME.xml"
222+
else
223+
sed -i 's# name="github.com/filecoin-project/lotus/\(.*\)"# name="'${{ matrix.name }}':\1"#g' "$REPORTS_PATH/$NAME.xml"
224+
fi
225+
cat "$REPORTS_PATH/$NAME.xml"
226+
- if: (!cancelled())
227+
uses: actions/upload-artifact@v4
228+
with:
229+
name: ${{ matrix.name }}-${{ runner.os }}-${{ runner.arch }}
230+
path: |
231+
${{ steps.reports.outputs.path }}/${{ matrix.name }}.xml
232+
${{ steps.reports.outputs.path }}/${{ matrix.name }}.json
233+
continue-on-error: true

‎.github/workflows/very-expensive-test.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Very Expensive Test
2+
3+
on:
4+
pull_request:
5+
types:
6+
- opened
7+
- synchronize
8+
- reopened
9+
# WARN: This will run very expensive test every time ANY label is added
10+
- labeled
11+
schedule:
12+
- cron: '0 0 * * *' # Runs nightly at 0AM UTC
13+
14+
permissions:
15+
contents: read
16+
issues: write
17+
pull-requests: write
18+
19+
jobs:
20+
test:
21+
name: Test
22+
if: github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'need/very-expensive-tests')
23+
uses: ./.github/workflows/reusable-test.yml
24+
with:
25+
run_very_expensive_tests: true
26+
issue:
27+
name: Issue
28+
if: failure() && github.event_name == 'schedule'
29+
needs: test
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: Create issue
33+
uses: ipdxco/create-or-update-issue@v1
34+
with:
35+
GITHUB_TOKEN: ${{ github.token }}
36+
title: Very expensive test run failed
37+
body: |
38+
The very expensive test run failed. See [the workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
39+
label: area/very-expensive-tests

‎cmd/ci/main.go‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type TestGroupExecutionContext struct {
1919
type Runner []string
2020

2121
var (
22+
linux_x64_5xlarge = []string{"self-hosted", "linux", "x64", "5xlarge"}
2223
linux_x64_4xlarge = []string{"self-hosted", "linux", "x64", "4xlarge"}
2324
linux_x64_2xlarge = []string{"self-hosted", "linux", "x64", "2xlarge"}
2425
linux_x64_xlarge = []string{"self-hosted", "linux", "x64", "xlarge"}
@@ -63,6 +64,12 @@ func main() {
6364
{
6465
Name: "list-test-group-execution-contexts",
6566
Usage: "List all test group execution contexts",
67+
Flags: []cli.Flag{
68+
&cli.BoolFlag{
69+
Name: "very-expensive-tests-run",
70+
Usage: "Whether to only include the groups with very expensive tests",
71+
},
72+
},
6673
Action: func(c *cli.Context) error {
6774
integrationTestGroups, err := getIntegrationTestGroups()
6875
if err != nil {
@@ -71,6 +78,15 @@ func main() {
7178
unitTestGroups := getUnitTestGroups()
7279
otherTestGroups := getOtherTestGroups()
7380
groups := append(append(integrationTestGroups, unitTestGroups...), otherTestGroups...)
81+
if c.Bool("very-expensive-tests-run") {
82+
var filteredGroups []TestGroupExecutionContext
83+
for _, group := range groups {
84+
if getHasVeryExpensiveTests(group.Name) {
85+
filteredGroups = append(filteredGroups, group)
86+
}
87+
}
88+
groups = filteredGroups
89+
}
7490
b, err := json.MarshalIndent(groups, "", " ")
7591
if err != nil {
7692
log.Fatal(err)
@@ -199,7 +215,7 @@ func getRunners(testGroupName string) []Runner {
199215
"itest-msgindex": {linux_x64_xlarge},
200216
"itest-multisig": {linux_x64_xlarge},
201217
"itest-net": {linux_x64_xlarge},
202-
"itest-niporep_manual": {linux_x64_4xlarge},
218+
"itest-niporep_manual": {linux_x64_5xlarge},
203219
"itest-nonce": {linux_x64_xlarge},
204220
"itest-path_detach_redeclare": {linux_x64_xlarge},
205221
"itest-pending_deal_allocation": {linux_x64_xlarge},
@@ -227,6 +243,13 @@ func getRunners(testGroupName string) []Runner {
227243
return []Runner{linux_x64}
228244
}
229245

246+
func getHasVeryExpensiveTests(testGroupName string) bool {
247+
testGroupNames := []string{
248+
"itest-niporep_manual",
249+
}
250+
return contains(testGroupNames, testGroupName)
251+
}
252+
230253
func getTestGroupMetadata(testGroupName string) TestGroupMetadata {
231254
packages := getPackages(testGroupName)
232255
needsParameters := getNeedsParameters(testGroupName)

0 commit comments

Comments
 (0)