diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4230068..2ca0e2d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,6 +76,19 @@ jobs: os: [ "ubuntu-latest" ] # Test against all security and bugfix versions: https://devguide.python.org/versions/ python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] + include: + # Test other OS's for a 'very stable' Python version too + - os: "windows-latest" + python-version: "3.11" + - os: "macos-latest" + python-version: "3.11" + # Include other OS for latest Python + # because these seem to be the flakiest from experience + # so are worth the extra testing + - os: "windows-latest" + python-version: "3.13" + - os: "macos-latest" + python-version: "3.13" runs-on: "${{ matrix.os }}" defaults: run: @@ -87,6 +100,15 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + - name: Install Fortran compiler - MacOS + # When building on Mac, we need to have a Fortran compiler + if: ${{ startsWith(matrix.os, 'macos') }} + uses: fortran-lang/setup-fortran@v1 + id: setup-fortran-macos + with: + # TODO: figure out whether we need/want to use other compilers too + compiler: "gcc" + version: "13" - uses: ./.github/actions/setup with: python-version: ${{ matrix.python-version }} @@ -96,10 +118,9 @@ jobs: # when people try to run without installing optional dependencies, # we should add a CI step that runs the tests without optional dependencies too. # We don't have that right now, because we're not sure this pain point exists. - uv-dependency-install-flags: "--all-extras --group tests" + uv-dependency-install-flags: "-v --reinstall-package example-fgen-basic -Ccompile-args='-v' --all-extras --group tests " - name: Run tests run: | - COV_DIR=`uv run --no-sync python -c 'from pathlib import Path; import example_fgen_basic; print(Path(example_fgen_basic.__file__).parent)'` uv run --no-sync pytest -r a -v tests src --doctest-modules --doctest-report ndiff --cov=${COV_DIR} --cov-report=term-missing --cov-report=xml uv run --no-sync coverage report @@ -218,6 +239,11 @@ jobs: check-build: strategy: matrix: + # The tests are also effectively a test of the build (at least the Fortran compilation). + # This test is really just about checking what is packaged + # and that we have all the required pieces. + # Hence, for now, only test on ubuntu + # (little value testing on other OS's because this test is not testing compilation directly). os: [ "ubuntu-latest" ] python-version: [ "3.11" ] runs-on: "${{ matrix.os }}" @@ -244,11 +270,16 @@ jobs: cd dist ls python3 -m venv venv - # TODO: alter for windows source venv/bin/activate pip install example_fgen_basic* python -c 'from example_fgen_basic.get_wavelength import get_wavelength_plain; print(get_wavelength_plain(23.4))' + # check-build-wheel + # For now, decide not to do this as odds of breaking the build are low + # (and we already implicitly test this when running the tests). + # If we find we are breaking the wheel build inadvertently often, + # we can always add this step in. + check-dependency-licences: strategy: matrix: diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index ef24f4c..7e881b0 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -24,19 +24,22 @@ jobs: # this permission is mandatory for trusted publishing with PyPI id-token: write steps: - - name: Check out repository + - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: ./.github/actions/setup + - name: Download artifacts + env: + GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" + run: | + gh release download --pattern "*.whl" --dir dist + gh release download --pattern "*.tar.gz" --dir dist + # Check what we have + tree dist + - name: Setup uv + id: setup-uv + uses: astral-sh/setup-uv@v4 with: - python-version: ${{ matrix.python-version }} - uv-dependency-install-flags: "--all-extras --group dev" + version: "0.8.8" - name: Publish to PyPI - run: | - # TODO: move to using cibuildwheel so we have wheels for multiple platforms and python versions - # starting docs: https://cibuildwheel.pypa.io/en/stable/ - uv build --sdist - uv publish - # Just in case, undo the changes to `pyproject.toml` - git restore --staged . && git restore . + run: uv publish diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 17dbefe..bea4913 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,8 +9,104 @@ defaults: shell: bash jobs: + build-wheels: + name: Build wheels on ${{ matrix.os }} for ${{ matrix.cibw_archs }} + runs-on: ${{ matrix.os }} + env: + # 3.9, 3.10 seem to not be happy on windows arm64 + # because you can't install numpy from a wheel for the build environment + # (and we're not going to try recompiling from source) + CIBW_SKIP: "cp39-win_arm64 cp310-win_arm64" + # TODO: consider turning this on + # CIBW_ENABLE: cpython-freethreading + CIBW_BEFORE_BUILD_WINDOWS: >- + pip install delvewheel + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- + delvewheel repair -w {dest_dir} {wheel} + CIBW_BEFORE_TEST: >- + pip install -r {project}/requirements-only-tests-min-locked.txt + CIBW_TEST_COMMAND: >- + pytest {project}/tests + MACOSX_DEPLOYMENT_TARGET: "13.0" + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + cibw_archs: "x86_64" + - os: ubuntu-24.04-arm + cibw_archs: "aarch64" + - os: windows-2025 + cibw_archs: "AMD64" + - os: windows-11-arm + cibw_archs: "ARM64" + - os: macos-13 + cibw_archs: "x86_64" + - os: macos-14 + cibw_archs: "arm64" + + steps: + - uses: actions/checkout@v5 + - name: Install Fortran compiler - MacOS + # When building on Mac, we need to have a Fortran compiler + if: ${{ startsWith(matrix.os, 'macos') }} + uses: fortran-lang/setup-fortran@v1 + id: setup-fortran-macos + with: + # TODO: figure out whether we need/want to use other compilers too + compiler: "gcc" + version: "13" + - name: Specify LLVM - windows-arm + if: ${{ matrix.os == 'windows-11-arm' }} + shell: pwsh + run: | + Invoke-WebRequest https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.8/LLVM-20.1.8-woa64.exe -OutFile LLVM-woa64.exe + Start-Process -FilePath ".\LLVM-woa64.exe" -ArgumentList "/S" -Wait + + $env:PATH = "C:\Program Files\LLVM\bin;" + $env:PATH + echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CC=clang-cl" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "CXX=clang-cl" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "FC=flang-new" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "AR=llvm-lib.exe" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Build wheels + uses: pypa/cibuildwheel@v3.1.4 # matplotlib had v3.1.3 + with: + package-dir: . + output-dir: wheelhouse + config-file: "{package}/pyproject.toml" + env: + CIBW_ARCHS: ${{ matrix.cibw_archs }} + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ matrix.cibw_archs }} + path: ./wheelhouse/*.whl + if-no-files-found: error + + make-sdist: + name: Make source distribution + runs-on: "ubuntu-latest" + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup + with: + python-version: ${{ matrix.python-version }} + uv-dependency-install-flags: "--all-extras --group dev" + - name: Create source distribution + run: uv build --sdist + - name: Upload the source distribution artefact + uses: actions/upload-artifact@v4 + with: + name: cibw-sdist + path: dist/*.tar.gz + draft-release: name: Create draft release + needs: [ build-wheels, make-sdist ] strategy: matrix: os: [ "ubuntu-latest" ] @@ -25,15 +121,6 @@ jobs: with: python-version: ${{ matrix.python-version }} uv-dependency-install-flags: "--all-extras --group dev" - - name: Add version to environment - run: | - PROJECT_VERSION=`sed -ne 's/^version = "\([0-9\.]*\)"/\1/p' pyproject.toml` - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV - - name: Build package for PyPI - run: | - uv build - # Just in case, undo the changes to `pyproject.toml` - git restore --staged . && git restore . - name: Generate Release Notes run: | echo "" >> ".github/release_template.md" @@ -44,6 +131,12 @@ jobs: echo "## Changes" >> ".github/release_template.md" echo "" >> ".github/release_template.md" git log $(git describe --tags --abbrev=0 HEAD^)..HEAD --pretty='format:* %h %s' --no-merges >> ".github/release_template.md" + - name: Download artifacts + uses: actions/download-artifact@v5 + with: + pattern: cibw-* + path: dist + merge-multiple: true - name: Create Release Draft uses: softprops/action-gh-release@v2 with: @@ -51,5 +144,5 @@ jobs: token: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" draft: true files: | - dist/example_fgen_basic-${{ env.PROJECT_VERSION }}-py3-none-any.whl - dist/example_fgen_basic-${{ env.PROJECT_VERSION }}.tar.gz + dist/example_fgen_basic-*.whl + dist/example_fgen_basic-*.tar.gz diff --git a/changelog/21.feature.md b/changelog/21.feature.md new file mode 100644 index 0000000..2328eae --- /dev/null +++ b/changelog/21.feature.md @@ -0,0 +1 @@ +Added compilation and publication to PyPI of wheels for major OS's and supported Python versions diff --git a/docs/development.md b/docs/development.md index 832dfe1..1b02821 100644 --- a/docs/development.md +++ b/docs/development.md @@ -51,6 +51,12 @@ that will be performed to be manually specified. See the `uv version` docs (specifically the `--bump` flag) for the [list of available bump rules](https://docs.astral.sh/uv/reference/cli/#uv-version). +Releasing also includes building of wheels for major operating systems and supported Python versions. +This is done thanks to [cibuildwheel](https://cibuildwheel.pypa.io/), an incredible service. +The wheel building can be quite tricky and fiddly. +If there are issues, we recommend starting with the [cibuildwheel](https://cibuildwheel.pypa.io/) docs +or raising an issue to discuss with the other maintainers. + ### Standard process The steps required are the following: @@ -65,7 +71,8 @@ The steps required are the following: (see here: [project releases](https://github.com/openscm/example-fgen-basic/releases)). Once you are happy with the release - (removed placeholders, added key announcements etc.) + (removed placeholders, added key announcements, + checked that all the expected wheels and source distribution is there etc.) then hit 'Publish release'. This triggers a release to PyPI (which you can then add to the release if you want). diff --git a/meson.build b/meson.build index 82c4120..93437af 100644 --- a/meson.build +++ b/meson.build @@ -68,17 +68,17 @@ if pyprojectwheelbuild_enabled 'src/example_fgen_basic/runtime_helpers.py', ) - # The ancillary library, - # i.e. all the stuff for wrapping that isn't directly exposed to Python. - ancillary_lib = library( - '@0@-ancillary'.format(meson.project_name()), - sources: srcs_ancillary_lib, - version: meson.project_version(), - dependencies: [], - # any other dependencies which aren't in source e.g. fgen-core - # e.g. dependencies: [fgen_core_dep], - install: false, - ) + # The ancillary library, + # i.e. all the stuff for wrapping that isn't directly exposed to Python. + ancillary_lib = library( + '@0@-ancillary'.format(meson.project_name()), + sources: srcs_ancillary_lib, + version: meson.project_version(), + dependencies: [], + # any other dependencies which aren't in source e.g. fgen-core + # e.g. dependencies: [fgen_core_dep], + install: false, + ) ancillary_dep = declare_dependency(link_with: ancillary_lib) diff --git a/pyproject.toml b/pyproject.toml index ac8fd09..0b402bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,18 +123,20 @@ all-dev = [ [build-system] build-backend = "mesonpy" requires = [ - "meson-python>=0.15.0", - "numpy", + "meson-python>=0.15.0", + # "numpy", + "numpy>=1.26.0; python_version < '3.13'", + "numpy>=2.1.0; python_version >= '3.13'", ] # https://mesonbuild.com/meson-python/how-to-guides/meson-args.html [tool.meson-python.args] setup = [ - '--default-library=static', - '-Dpyprojectwheelbuild=enabled', - ] + '--default-library=static', + '-Dpyprojectwheelbuild=enabled', +] install = [ -'--skip-subprojects', + '--skip-subprojects', ] [tool.coverage.run]