283 lines (270 loc) · 12.1 KB
/
test_upb.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
name: μpb Tests
on:
workflow_call:
inputs:
safe-checkout:
required: true
description: "The SHA key for the commit we want to run over"
type: string
permissions:
contents: read
jobs:
linux-clang:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
config:
- { name: "Bazel 7", bazel_version: "7.1.1" }
- { name: "Fastbuild" }
- { name: "Optimized", flags: "-c opt" }
- { name: "ASAN", flags: "--config=asan -c dbg", exclude-targets: "-//benchmarks:benchmark -//python/...", runner: ubuntu-20-large }
- { name: "UBSAN", flags: "--config=ubsan -c dbg", exclude-targets: "-//benchmarks:benchmark -//python/... -//lua/..." }
- { name: "32-bit", flags: "--copt=-m32 --linkopt=-m32", exclude-targets: "-//benchmarks:benchmark -//python/..." }
# TODO: Add 32-bit ASAN test
# TODO: Restore the FastTable tests
name: ${{ matrix.config.name }}
runs-on: ${{ matrix.config.runner || 'ubuntu-latest' }}
steps:
- name: Checkout pending changes
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v3
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '6.3.0' }}-75f2a85ece6526cc3d54087018c0f1097d78d42b
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: upb-bazel
bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //protos/... //protos_generator/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
exclude-targets: ${{ matrix.config.exclude-targets }}
linux-gcc:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
name: GCC Optimized
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v3
with:
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17"
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-gcc"
bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt //bazel/... //benchmarks/... //lua/... //protos/... //protos_generator/... //python/... //upb/... //upb_generator/...
windows:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
name: Windows
runs-on: windows-2022
steps:
- name: Checkout pending changes
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
cache: pip
cache-dependency-path: 'python/requirements.txt'
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel@v3
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-windows"
bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/... //protos/... //protos_generator/...
version: 6.3.0
exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test
macos:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
config:
- { name: "macOS", bazel-command: "test" }
- { name: "macOS ARM (build only)", bazel-command: "build", flags: "--cpu=darwin_arm64" }
name: ${{ matrix.config.name }}
runs-on: macos-12
steps:
- name: Checkout pending changes
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
cache: pip
cache-dependency-path: 'python/requirements.txt'
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel@v3
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-macos"
bazel: ${{ matrix.config.bazel-command }} --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //protos/... //protos_generator/... //python/... //upb/... //upb_generator/...
version: 6.3.0
no-python:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
name: No System Python
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v3
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-no-python"
bash: >-
which python3 &&
mv `which python3` /tmp &&
! which python3 &&
bazel test $BAZEL_FLAGS --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //python/... -- -//python/dist:source_wheel
build_wheels:
name: Build Wheels
runs-on: ubuntu-latest
if: ${{ github.event_name != 'pull_request_target' }}
steps:
- name: Checkout pending changes
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
- name: Build Wheels
uses: protocolbuffers/protobuf-ci/bazel-docker@v3
with:
image: us-docker.pkg.dev/protobuf-build/release-containers/linux/apple@sha256:b3dc9b75d8e599b0e95ed245d89f44b5a4231112f975da89dd02006a484a58df
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: upb-bazel-python
bazel: build --crosstool_top=//toolchain:clang_suite --//toolchain:release=true --symlink_prefix=/ -c dbg //python/dist //python/dist:test_wheel //python/dist:source_wheel
- name: Move Wheels
run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf-*.tar.gz' \) -exec mv '{}' wheels ';'
- uses: actions/upload-artifact@v3
with:
name: python-wheels
path: wheels/
- uses: actions/upload-artifact@v3
with:
name: requirements
# Tests shouldn't have access to the whole upb repo, upload the one file we need
path: python/requirements.txt
test_wheels:
name: Test Wheels
needs: build_wheels
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
include:
# Linux and Mac use the limited API, so all Python versions will use
# a single wheel. As a result we can just test the oldest and newest
# supported Python versions and assume this gives us sufficient test
# coverage.
- { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'binary' }
- { os: macos-11, python-version: "3.8", architecture: x64, type: 'binary' }
- { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'binary' }
- { os: macos-12, python-version: "3.12", architecture: x64, type: 'binary' }
- { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'source' }
- { os: macos-11, python-version: "3.8", architecture: x64, type: 'source' }
- { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'source' }
- { os: macos-12, python-version: "3.12", architecture: x64, type: 'source' }
# Windows uses the full API up until Python 3.10.
- { os: windows-2019, python-version: "3.8", architecture: x86, type: 'binary' }
- { os: windows-2019, python-version: "3.9", architecture: x86, type: 'binary' }
- { os: windows-2019, python-version: "3.10", architecture: x86, type: 'binary' }
- { os: windows-2019, python-version: "3.11", architecture: x86, type: 'binary' }
- { os: windows-2019, python-version: "3.12", architecture: x86, type: 'binary' }
- { os: windows-2019, python-version: "3.8", architecture: x64, type: 'binary' }
- { os: windows-2019, python-version: "3.9", architecture: x64, type: 'binary' }
- { os: windows-2019, python-version: "3.10", architecture: x64, type: 'binary' }
- { os: windows-2019, python-version: "3.11", architecture: x64, type: 'binary' }
- { os: windows-2019, python-version: "3.12", architecture: x64, type: 'binary' }
runs-on: ${{ matrix.os }}
if: ${{ github.event_name != 'pull_request_target' }}
defaults:
run:
shell: bash
steps:
- name: Download Wheels
uses: actions/download-artifact@v3
with:
name: python-wheels
path: wheels
- name: Download Requirements
uses: actions/download-artifact@v3
with:
name: requirements
path: requirements
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: ${{ matrix.python-version }}
architecture: ${{ matrix.architecture }}
- name: Setup Python venv
run: |
python -m pip install --upgrade pip
python -m venv env
# Windows uses 'Scripts' instead of 'bin'
source env/bin/activate || source env/Scripts/activate
echo "VIRTUAL ENV:" $VIRTUAL_ENV
- name: Install tzdata
run: pip install tzdata
# Only needed on Windows, Linux ships with tzdata.
if: ${{ contains(matrix.os, 'windows') }}
- name: Install requirements
run: pip install -r requirements/requirements.txt
- name: Install Protobuf Binary Wheel
run: pip install -vvv --no-index --find-links wheels protobuf
if: ${{ matrix.type == 'binary' }}
- name: Install Protobuf Source Wheel
run: |
cd wheels
tar -xzvf *.tar.gz
cd protobuf-*/
pip install .
if: ${{ matrix.type == 'source' }}
- name: Test that module is importable
run: python -v -c 'from google._upb import _message; assert "google._upb._message.MessageMeta" in str(_message.MessageMeta)'
- name: Install Protobuf Test Wheel
run: pip install -vvv --no-index --find-links wheels protobuftests
- name: Run the unit tests
run: |
TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g')
for test in $TESTS; do
python -m unittest -v $test
done
test_pure_python_wheels:
name: Test Pure Python Wheels
needs: build_wheels
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
python-version: ["3.8", "3.12"]
runs-on: ubuntu-latest
if: ${{ github.event_name != 'pull_request_target' }}
steps:
- name: Download Wheels
uses: actions/download-artifact@v3
with:
name: python-wheels
path: wheels
- name: Delete Binary Wheels
run: find wheels -type f | grep -v none-any | xargs rm
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: ${{ matrix.python-version }}
- name: Setup Python venv
run: |
python -m pip install --upgrade pip
python -m venv env
source env/bin/activate
echo "VIRTUAL ENV:" $VIRTUAL_ENV
- name: Install numpy
run: pip install numpy
- name: Install Protobuf Wheels
run: pip install -vvv --no-index --find-links wheels protobuf protobuftests
- name: Run the unit tests
run: |
TESTS=$(pip show -f protobuftests | grep _test.py | sed 's,/,.,g' | sed -E 's,.py$,,g')
for test in $TESTS; do
python -m unittest -v $test
done