Skip to content

Commit 12e3723

Browse files
GregoryComerGregoryComer
committed
[Backend Tester] Pick test jobs + logic onto release/1.0
1 parent 88e0855 commit 12e3723

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+637
-103
lines changed

‎.ci/scripts/test_backend_linux.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,17 @@ if [[ "$FLOW" == *qnn* ]]; then
3939
fi
4040

4141
if [[ "$FLOW" == *vulkan* ]]; then
42-
# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
42+
# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate.
4343
source .ci/scripts/setup-vulkan-linux-deps.sh
4444

4545
EXTRA_BUILD_ARGS+=" -DEXECUTORCH_BUILD_VULKAN=ON"
4646
fi
4747

48+
if [[ "$FLOW" == *arm* ]]; then
49+
# Setup ARM deps.
50+
.ci/scripts/setup-arm-baremetal-tools.sh
51+
fi
52+
4853
# We need the runner to test the built library.
4954
PYTHON_EXECUTABLE=python CMAKE_ARGS="$EXTRA_BUILD_ARGS" .ci/scripts/setup-linux.sh --build-tool cmake --build-mode Release --editable true
5055

‎.github/workflows/_test_backend.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: Test Backend
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
backend:
7+
description: 'Backend to test (xnnpack, coreml, vulkan, qnn)'
8+
required: true
9+
type: string
10+
flows:
11+
description: 'JSON array of flows to test'
12+
required: true
13+
type: string
14+
ref:
15+
description: 'Git ref to checkout'
16+
required: false
17+
type: string
18+
default: ${{ github.sha }}
19+
timeout:
20+
description: 'Job timeout in minutes'
21+
required: false
22+
type: number
23+
default: 120
24+
run-linux:
25+
description: 'Whether to run Linux tests'
26+
required: false
27+
type: boolean
28+
default: false
29+
run-macos:
30+
description: 'Whether to run macOS tests'
31+
required: false
32+
type: boolean
33+
default: false
34+
runner-linux:
35+
description: 'Runner type for Linux jobs'
36+
required: false
37+
type: string
38+
default: linux.4xlarge.memory
39+
40+
jobs:
41+
test-backend-linux:
42+
if: ${{ inputs.run-linux }}
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
flow: ${{ fromJSON(inputs.flows) }}
47+
suite: [models, operators]
48+
49+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@release/2.9
50+
with:
51+
ref: ${{ inputs.ref }}
52+
runner: ${{ inputs.runner-linux }}
53+
docker-image: ci-image:executorch-ubuntu-22.04-clang12
54+
submodules: recursive
55+
timeout: ${{ inputs.timeout }}
56+
upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }}
57+
script: |
58+
set -eux
59+
60+
source .ci/scripts/test_backend_linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}"
61+
62+
test-backend-macos:
63+
if: ${{ inputs.run-macos }}
64+
strategy:
65+
fail-fast: false
66+
matrix:
67+
flow: ${{ fromJSON(inputs.flows) }}
68+
suite: [models, operators]
69+
70+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@release/2.9
71+
with:
72+
ref: ${{ inputs.ref }}
73+
runner: macos-m1-stable
74+
python-version: "3.12"
75+
submodules: recursive
76+
timeout: ${{ inputs.timeout }}
77+
upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }}
78+
script: |
79+
set -eux
80+
81+
# This is needed to get the prebuilt PyTorch wheel from S3
82+
${CONDA_RUN} --no-capture-output pip install awscli==1.37.21
83+
84+
source .ci/scripts/test_backend_macos.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}"

‎.github/workflows/nightly.yml‎

Lines changed: 1 addition & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
repo-name: pytorch
2828
branch: main
2929
pin-folder: .ci/docker/ci_commit_pins
30-
test-infra-ref: release/2.9
30+
test-infra-ref: main
3131
updatebot-token: ${{ secrets.UPDATEBOT_TOKEN }}
3232
pytorchbot-token: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
3333

@@ -36,51 +36,3 @@ jobs:
3636
uses: ./.github/workflows/_link_check.yml
3737
with:
3838
ref: ${{ github.sha }}
39-
40-
backend-test-linux:
41-
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@release/2.9
42-
strategy:
43-
fail-fast: false
44-
matrix:
45-
flow: [
46-
qnn, qnn_16a16w, qnn_16a8w, qnn_16a4w, qnn_16a4w_block, qnn_8a8w,
47-
vulkan, vulkan_static_int8_per_channel,
48-
xnnpack, xnnpack_dynamic_int8_per_channel, xnnpack_static_int8_per_channel, xnnpack_static_int8_per_tensor
49-
]
50-
suite: [models, operators]
51-
with:
52-
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
53-
runner: linux.4xlarge.memory
54-
docker-image: ci-image:executorch-ubuntu-22.04-clang12
55-
submodules: recursive
56-
timeout: 120
57-
upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }}
58-
script: |
59-
set -eux
60-
61-
source .ci/scripts/test_backend_linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}"
62-
63-
backend-test-macos:
64-
uses: pytorch/test-infra/.github/workflows/macos_job.yml@release/2.9
65-
permissions:
66-
id-token: write
67-
contents: read
68-
strategy:
69-
fail-fast: false
70-
matrix:
71-
flow: [coreml, coreml_static_int8]
72-
suite: [models, operators]
73-
with:
74-
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
75-
runner: macos-m1-stable
76-
python-version: 3.12
77-
submodules: recursive
78-
timeout: 120
79-
upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }}
80-
script: |
81-
set -eux
82-
83-
# This is needed to get the prebuilt PyTorch wheel from S3
84-
${CONDA_RUN} --no-capture-output pip install awscli==1.37.21
85-
86-
source .ci/scripts/test_backend_macos.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}"

‎.github/workflows/test-backend-arm.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: Test ARM Backend
2+
3+
on:
4+
schedule:
5+
- cron: 0 2 * * *
6+
push:
7+
branches:
8+
- release/*
9+
tags:
10+
- ciflow/nightly/*
11+
pull_request:
12+
paths:
13+
- .github/workflows/test-backend-arm.yml
14+
- .github/workflows/_test_backend.yml
15+
workflow_dispatch:
16+
17+
concurrency:
18+
group: ${{ github.workflow }}--${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
19+
cancel-in-progress: true
20+
21+
jobs:
22+
test-arm:
23+
uses: ./.github/workflows/_test_backend.yml
24+
with:
25+
backend: arm
26+
flows: '["arm_tosa"]'
27+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
28+
timeout: 120
29+
run-linux: true

‎.github/workflows/test-backend-coreml.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: Test CoreML Backend
2+
3+
on:
4+
schedule:
5+
- cron: 0 2 * * *
6+
push:
7+
branches:
8+
- release/*
9+
tags:
10+
- ciflow/nightly/*
11+
pull_request:
12+
paths:
13+
- .github/workflows/test-backend-coreml.yml
14+
- .github/workflows/_test_backend.yml
15+
workflow_dispatch:
16+
17+
concurrency:
18+
group: ${{ github.workflow }}--${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
19+
cancel-in-progress: true
20+
21+
jobs:
22+
test-coreml:
23+
uses: ./.github/workflows/_test_backend.yml
24+
with:
25+
backend: coreml
26+
flows: '["coreml", "coreml_static_int8"]'
27+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
28+
timeout: 120
29+
run-macos: true

‎.github/workflows/test-backend-qnn.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Test QNN Backend
2+
3+
on:
4+
schedule:
5+
- cron: 0 2 * * *
6+
push:
7+
branches:
8+
- release/*
9+
tags:
10+
- ciflow/nightly/*
11+
pull_request:
12+
paths:
13+
- .github/workflows/test-backend-qnn.yml
14+
- .github/workflows/_test_backend.yml
15+
workflow_dispatch:
16+
17+
concurrency:
18+
group: ${{ github.workflow }}--${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
19+
cancel-in-progress: true
20+
21+
jobs:
22+
test-qnn:
23+
uses: ./.github/workflows/_test_backend.yml
24+
with:
25+
backend: qnn
26+
flows: '["qnn", "qnn_16a16w", "qnn_16a8w", "qnn_16a4w", "qnn_16a4w_block", "qnn_8a8w"]'
27+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
28+
timeout: 120
29+
run-linux: true
30+
runner-linux: linux.8xlarge.memory

‎.github/workflows/test-backend-vulkan.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: Test Vulkan Backend
2+
3+
on:
4+
schedule:
5+
- cron: 0 2 * * *
6+
push:
7+
branches:
8+
- release/*
9+
tags:
10+
- ciflow/nightly/*
11+
pull_request:
12+
paths:
13+
- .github/workflows/test-backend-vulkan.yml
14+
- .github/workflows/_test_backend.yml
15+
workflow_dispatch:
16+
17+
concurrency:
18+
group: ${{ github.workflow }}--${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
19+
cancel-in-progress: true
20+
21+
jobs:
22+
test-vulkan:
23+
uses: ./.github/workflows/_test_backend.yml
24+
with:
25+
backend: vulkan
26+
flows: '["vulkan", "vulkan_static_int8_per_channel"]'
27+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
28+
timeout: 120
29+
run-linux: true

‎.github/workflows/test-backend-xnnpack.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: Test XNNPACK Backend
2+
3+
on:
4+
schedule:
5+
- cron: 0 2 * * *
6+
push:
7+
branches:
8+
- release/*
9+
tags:
10+
- ciflow/nightly/*
11+
pull_request:
12+
paths:
13+
- .github/workflows/test-backend-xnnpack.yml
14+
- .github/workflows/_test_backend.yml
15+
workflow_dispatch:
16+
17+
concurrency:
18+
group: ${{ github.workflow }}--${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
19+
cancel-in-progress: true
20+
21+
jobs:
22+
test-xnnpack:
23+
uses: ./.github/workflows/_test_backend.yml
24+
with:
25+
backend: xnnpack
26+
flows: '["xnnpack", "xnnpack_dynamic_int8_per_channel", "xnnpack_static_int8_per_channel", "xnnpack_static_int8_per_tensor"]'
27+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
28+
timeout: 120
29+
run-linux: true

‎.github/workflows/test-cuda-builds.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Test ExecuTorch CUDA Build Compatibility
2+
# This workflow tests whether ExecuTorch can be successfully built with CUDA support
3+
# across different CUDA versions (12.6, 12.8, 12.9) using the command:
4+
# CMAKE_ARGS="-DEXECUTORCH_BUILD_CUDA=ON" ./install_executorch.sh
5+
#
6+
# Note: ExecuTorch automatically detects the system CUDA version using nvcc and
7+
# installs the appropriate PyTorch wheel. No manual CUDA/PyTorch installation needed.
8+
9+
name: Test CUDA Builds
10+
11+
on:
12+
pull_request:
13+
push:
14+
branches:
15+
- main
16+
- release/*
17+
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
20+
cancel-in-progress: false
21+
22+
jobs:
23+
test-cuda-builds:
24+
strategy:
25+
fail-fast: false
26+
matrix:
27+
cuda-version: ["12.6", "12.8", "13.0"]
28+
29+
name: test-executorch-cuda-build-${{ matrix.cuda-version }}
30+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
31+
permissions:
32+
id-token: write
33+
contents: read
34+
with:
35+
timeout: 90
36+
runner: linux.g5.4xlarge.nvidia.gpu
37+
gpu-arch-type: cuda
38+
gpu-arch-version: ${{ matrix.cuda-version }}
39+
use-custom-docker-registry: false
40+
submodules: recursive
41+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
42+
script: |
43+
set -eux
44+
45+
# Test ExecuTorch CUDA build - ExecuTorch will automatically detect CUDA version
46+
# and install the appropriate PyTorch wheel when CMAKE_ARGS="-DEXECUTORCH_BUILD_CUDA=ON"
47+
source .ci/scripts/test-cuda-build.sh "${{ matrix.cuda-version }}"
48+
49+
# This job will fail if any of the CUDA versions fail
50+
check-all-cuda-builds:
51+
needs: test-cuda-builds
52+
runs-on: ubuntu-latest
53+
if: always()
54+
steps:
55+
- name: Check if all CUDA builds succeeded
56+
run: |
57+
if [[ "${{ needs.test-cuda-builds.result }}" != "success" ]]; then
58+
echo "ERROR: One or more ExecuTorch CUDA builds failed!"
59+
echo "CUDA build results: ${{ needs.test-cuda-builds.result }}"
60+
exit 1
61+
else
62+
echo "SUCCESS: All ExecuTorch CUDA builds (12.6, 12.8, 12.9) completed successfully!"
63+
fi

0 commit comments

Comments
 (0)