PGO build for MacOS M1 #3893
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - '**' | |
| pull_request: {} | |
| env: | |
| COLUMNS: 150 | |
| jobs: | |
| coverage: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| - id: cache-rust | |
| name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: coverage-v2 | |
| - run: cargo install rustfilt coverage-prepare | |
| if: steps.cache-rust.outputs.cache-hit != 'true' | |
| - run: rustup component add llvm-tools-preview | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - run: pip install -r tests/requirements.txt | |
| - run: rustc --version --verbose | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: '-C instrument-coverage' | |
| - run: pip freeze | |
| - run: coverage run -m pytest | |
| - run: ls -lha | |
| - run: coverage xml | |
| - run: coverage-prepare lcov python/pydantic_core/*.so | |
| - uses: codecov/codecov-action@v3 | |
| # See https://github.com/PyO3/pyo3/discussions/2781 | |
| # tests intermittently segfault with pypy and cpython 3.7 when using `coverage run ...`, hence separate job | |
| test-python: | |
| name: test ${{ matrix.python-version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - '3.7' | |
| - '3.8' | |
| - '3.9' | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| - 'pypy3.7' | |
| - 'pypy3.8' | |
| - 'pypy3.9' | |
| - 'pypy3.10' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-v3 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: pip freeze | |
| - run: pytest | |
| env: | |
| HYPOTHESIS_PROFILE: slow | |
| test-os: | |
| name: test on ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ matrix.os }}-v1 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: pip freeze | |
| - run: pytest | |
| - run: cargo test | |
| # test with a debug build as it picks up errors which optimised release builds do not | |
| test-debug: | |
| name: test-debug ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - '3.11' | |
| - 'pypy3.10' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-debug | |
| - run: pip install -r tests/requirements.txt | |
| - run: make build-dev | |
| - run: pip freeze | |
| - run: pytest | |
| test-pydantic-integration: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: pydantic/pydantic | |
| path: pydantic | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: pydantic-core | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-pydantic-integration | |
| - name: install deps | |
| run: | | |
| pip install pdm maturin | |
| pdm venv create --with-pip | |
| pdm install -G testing -G email | |
| pdm run pip install maturin | |
| pdm run bash -c 'cd ../pydantic-core && make build-dev' | |
| working-directory: pydantic | |
| - run: pdm info && pdm list | |
| working-directory: pydantic | |
| - run: pdm run pytest | |
| working-directory: pydantic | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| # used to lint js code | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - uses: actions/cache@v3 | |
| id: cache-py | |
| name: cache python | |
| with: | |
| path: ${{ env.pythonLocation }} | |
| key: > | |
| py | |
| ${{ env.pythonLocation }} | |
| ${{ hashFiles('tests/requirements-linting.txt') }} | |
| ${{ hashFiles('pyproject.toml') }} | |
| - run: pip install -r tests/requirements-linting.txt | |
| if: steps.cache-py.outputs.cache-hit != 'true' | |
| - run: make build-dev | |
| - run: pip freeze | |
| - run: make lint | |
| - run: make pyright | |
| - run: npm install | |
| - run: npm run lint | |
| bench: | |
| name: rust benchmarks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - run: pip install typing_extensions | |
| - run: cargo bench | |
| build-wasm-emscripten: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| components: rust-src | |
| targets: wasm32-unknown-emscripten | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: mymindstorm/setup-emsdk@v12 | |
| with: | |
| # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version | |
| # in node_modules/pyodide/repodata.json, to get the version, run: | |
| # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform` | |
| version: '3.1.32' | |
| actions-cache-folder: emsdk-cache | |
| - run: pip install 'maturin>=1,<2' 'black>=22.3.0,<23' typing_extensions | |
| - name: build wheels | |
| run: make build-wasm | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - run: npm install | |
| - run: npm run test | |
| - run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: wasm_wheels | |
| path: dist | |
| # https://github.com/marketplace/actions/alls-green#why used for branch protection checks | |
| check: | |
| if: always() | |
| needs: [coverage, test-python, test-os, test-debug, lint, bench, build-wasm-emscripten] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| allowed-failures: coverage | |
| build-sdist: | |
| name: build sdist | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| rust-toolchain: stable | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| build: | |
| name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| target: [x86_64, aarch64] | |
| manylinux: [auto] | |
| include: | |
| - os: ubuntu | |
| platform: linux | |
| - os: windows | |
| ls: dir | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10 | |
| - os: windows | |
| ls: dir | |
| target: i686 | |
| python-architecture: x86 | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| - os: windows | |
| ls: dir | |
| target: aarch64 | |
| interpreter: 3.11 3.12 | |
| - os: macos | |
| target: aarch64 | |
| interpreter: 3.7 3.8 3.9 pypy3.8 pypy3.9 pypy3.10 | |
| - os: ubuntu | |
| platform: linux | |
| target: i686 | |
| - os: ubuntu | |
| platform: linux | |
| target: aarch64 | |
| - os: ubuntu | |
| platform: linux | |
| target: armv7 | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| # musllinux | |
| - os: ubuntu | |
| platform: linux | |
| target: x86_64 | |
| manylinux: musllinux_1_1 | |
| - os: ubuntu | |
| platform: linux | |
| target: aarch64 | |
| manylinux: musllinux_1_1 | |
| - os: ubuntu | |
| platform: linux | |
| target: ppc64le | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| - os: ubuntu | |
| platform: linux | |
| target: s390x | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| exclude: | |
| # Optimized PGO builds for x86_64 manylinux and windows follow a different matrix, | |
| # maybe in future maturin-action can support this automatically | |
| - os: ubuntu | |
| target: x86_64 | |
| manylinux: auto | |
| - os: windows | |
| target: x86_64 | |
| # Windows on arm64 only supports Python 3.11+ | |
| - os: windows | |
| target: aarch64 | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| architecture: ${{ matrix.python-architecture || 'x64' }} | |
| - run: pip install -U twine 'black>=22.3.0,<23' typing_extensions | |
| # generate self-schema now, so we don't have to do so inside docker in maturin build | |
| - run: python generate_self_schema.py | |
| - name: build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: ${{ matrix.manylinux || 'auto' }} | |
| container: ${{ matrix.container }} | |
| args: --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }} ${{ matrix.extra-build-args }} | |
| rust-toolchain: stable | |
| docker-options: -e CI | |
| - run: ${{ matrix.ls || 'ls -lh' }} dist/ | |
| - run: twine check --strict dist/* | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| build-pgo: | |
| name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }} | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest-xlarge] | |
| interpreter: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] | |
| include: | |
| - os: windows-latest | |
| ls: dir | |
| exclude: | |
| - os: macos-latest-xlarge | |
| interpreter: '3.7' | |
| - os: macos-latest-xlarge | |
| interpreter: '3.8' | |
| - os: macos-latest-xlarge | |
| interpreter: '3.9' | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.interpreter }} | |
| - name: install rust stable | |
| id: rust-toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: llvm-tools | |
| - run: pip install -U 'black>=22.3.0,<23' typing_extensions | |
| # generate self-schema now, so we don't have to do so inside docker in maturin build | |
| - run: python generate_self_schema.py | |
| - run: rustc --version --verbose | |
| - name: build initial wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| args: > | |
| --release | |
| --out pgo-wheel | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: stable | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata" | |
| - name: detect rust host | |
| run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: generate pgo data | |
| run: | | |
| pip install -U pip | |
| pip install -r tests/requirements.txt | |
| pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall | |
| pytest tests/benchmarks | |
| rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"' | |
| - name: merge pgo data | |
| run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata | |
| - name: build pgo-optimized wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| args: > | |
| --release | |
| --out dist | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: stable | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata" | |
| - run: ${{ matrix.ls || 'ls -lh' }} dist/ | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| inspect-pypi-assets: | |
| needs: [build, build-sdist, build-pgo] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: list dist files before PGO builds | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - name: list dist files with PGO builds | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: extract and list sdist file | |
| run: | | |
| mkdir sdist-files | |
| tar -xvf dist/*.tar.gz -C sdist-files | |
| tree -a sdist-files | |
| - name: extract and list wheel file | |
| run: | | |
| ls dist/*cp310-manylinux*x86_64.whl | head -n 1 | |
| python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1` | |
| test-builds-arch: | |
| name: test build on ${{ matrix.target }}-${{ matrix.distro }} | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: [aarch64, armv7, s390x, ppc64le] | |
| distro: ['ubuntu22.04'] | |
| include: | |
| - target: aarch64 | |
| distro: alpine_latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - uses: uraimo/run-on-arch-action@v2.5.1 | |
| name: install & test | |
| with: | |
| arch: ${{ matrix.target }} | |
| distro: ${{ matrix.distro }} | |
| githubToken: ${{ github.token }} | |
| install: | | |
| set -x | |
| if command -v apt-get &> /dev/null; then | |
| echo "installing python & pip with apt-get..." | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-pip python3-venv | |
| else | |
| echo "installing python & pip with apk..." | |
| apk update | |
| apk add python3 py3-pip | |
| fi | |
| run: | | |
| set -x | |
| # typing-extensions isn't automatically installed because of `--no-index --no-deps` | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| python3 -m pip install -U pip typing-extensions | |
| python3 -m pip install -r tests/requirements.txt | |
| python3 -m pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -m pytest --ignore=tests/test_docstrings.py | |
| python3 -c 'import pydantic_core._pydantic_core; print(pydantic_core._pydantic_core.__version__)' | |
| test-builds-os: | |
| name: test build on ${{ matrix.os }} | |
| needs: [build, build-pgo] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - run: pip install typing-extensions | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall | |
| - run: pytest --ignore=tests/test_docstrings.py | |
| release: | |
| needs: [test-builds-arch, test-builds-os, build-sdist, check] | |
| if: success() && startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - run: pip install -U twine | |
| - name: check package version | |
| run: python .github/check_version.py | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - run: twine check --strict dist/* | |
| - name: upload to pypi | |
| run: twine upload dist/* | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.pypi_token }} | |
| - name: get wasm dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wasm_wheels | |
| path: wasm | |
| - name: upload to github release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: | | |
| wasm/*.whl | |
| prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }} |