ci.yml

name: ci

on:
  push:
    branches:
      - main
    tags:
      - '**'
  pull_request:
    types:
    - opened
    - synchronize
    - reopened
    - closed

concurrency:
  group: ${{ github.workflow }}/${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  coverage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: "pip"
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
          profile: minimal
          components: llvm-tools-preview
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov
      - uses: Swatinem/rust-cache@v2
        with:
          key: coverage-cargo-ubuntu
        continue-on-error: true
      - name: Setup virtual environment
        run: |
          python -m venv venv
          source venv/bin/activate
          pip install -r bindings/requirements-dev.txt
          pip install maturin
      - name: Run coverage
        run: |
          source venv/bin/activate
          cd bindings
          source <(cargo llvm-cov show-env --export-prefix)
          export CARGO_TARGET_DIR=$CARGO_LLVM_COV_TARGET_DIR
          export CARGO_INCREMENTAL=1
          cargo llvm-cov clean --workspace
          cargo check
          cargo test
          maturin develop
          pytest tests --cov=powerboxes --cov-report xml -m "not benchmark"
          cargo llvm-cov --no-run --lcov --output-path coverage-bindings.lcov
          cd ../powerboxesrs
          source <(cargo llvm-cov show-env --export-prefix)
          export CARGO_TARGET_DIR=$CARGO_LLVM_COV_TARGET_DIR
          cargo llvm-cov clean --workspace
          cargo check
          cargo test
          cargo llvm-cov --no-run --lcov --output-path coverage.lcov
      - uses: codecov/codecov-action@v3
        env:
          TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          files: coverage.lcov, coverage-bindings.lcov,coverage.xml
          name: ubuntu
  test-python:
    name: test ${{ matrix.python-version }}
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - '3.7'
          - '3.8'
          - '3.9'
          - '3.10'
          - '3.11'
          - '3.12'
          - 'pypy3.7'
          - 'pypy3.8'
          - 'pypy3.9'
          - 'pypy3.10'

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable

      - name: cache rust
        uses: Swatinem/rust-cache@v2
        with:
          key: test-v3

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - run: pip install -r bindings/requirements-dev.txt

      - run: pip install -e bindings/.
        env:
          RUST_BACKTRACE: 1

      - run: pip freeze

      - run: pytest bindings/tests -m "not benchmark"

  test-os:
    name: test on ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]

    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v4

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable

      - name: cache rust
        uses: Swatinem/rust-cache@v2
        with:
          key: ${{ matrix.os }}-v1

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - run: pip install -r bindings/requirements-dev.txt

      - run: pip install -e bindings/.
        env:
          RUST_BACKTRACE: 1

      - run: pip freeze

      - run: pytest -m "not benchmark"

      - run: cd powerboxesrs && cargo test && cd ../bindings && cargo test

  # https://github.com/marketplace/actions/alls-green#why used for branch protection checks
  check:
    if: always()
    needs: [coverage, test-python, test-os]
    runs-on: ubuntu-latest
    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}
          allowed-failures: coverage

  build-sdist:
    name: build sdist
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: --out dist
          rust-toolchain: stable
          working-directory: bindings
      - uses: actions/upload-artifact@v3
        with:
          name: pypi_files
          path: bindings/dist

  build:
    name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }})
    # only run on push to main and on release
    if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
    strategy:
      fail-fast: false
      matrix:
        os: [linux, macos, windows]
        target: [x86_64, aarch64]
        manylinux: [auto]
        include:
          # manylinux for various platforms, plus x86_64 pypy
          - os: linux
            manylinux: auto
            target: i686
          - os: linux
            manylinux: auto
            target: aarch64
          - os: linux
            manylinux: auto
            target: armv7
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          - os: linux
            manylinux: auto
            target: ppc64le
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          - os: linux
            manylinux: auto
            target: s390x
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          - os: linux
            manylinux: auto
            target: x86_64
            interpreter: pypy3.7 pypy3.8 pypy3.9 pypy3.10

          # musllinux
          - os: linux
            manylinux: musllinux_1_1
            target: x86_64
          - os: linux
            manylinux: musllinux_1_1
            target: aarch64

          # macos;
          # all versions x86_64
          # arm pypy and older pythons which can't be run on the arm hardware for PGO
          - os: macos
            target: x86_64
          - os: macos
            target: aarch64
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10

          # windows;
          - os: windows
            target: x86_64
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          - os: windows
            target: i686
            python-architecture: x86
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
        exclude:
          - os: windows
            target: aarch64

    runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest
    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          architecture: ${{ matrix.python-architecture || 'x64' }}

      - run: pip install -U twine

      - name: build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: ${{ matrix.manylinux == 'manylinux' && 'auto' || matrix.manylinux }}
          args: --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }}
          rust-toolchain: stable
          docker-options: -e CI
          working-directory: bindings

      - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} bindings/dist/

      - run: twine check --strict bindings/dist/*

      - uses: actions/upload-artifact@v3
        with:
          name: pypi_files
          path: bindings/dist

  build-pgo:
    name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }}
    # only run on push to main and on release
    if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
    strategy:
      fail-fast: false
      matrix:
        os: [linux, windows, macos]
        interpreter: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
        include:
          # standard runners with override for macos arm
          - os: linux
            runs-on: ubuntu-latest
          - os: windows
            ls: dir
            runs-on: windows-latest
          - os: macos
            runs-on: macos-latest
        exclude:
          # macos arm only supported from 3.10 and up
          - os: macos
            interpreter: '3.7'
          - os: macos
            interpreter: '3.8'
          - os: macos
            interpreter: '3.9'

    runs-on: ${{ matrix.runs-on }}
    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.interpreter }}

      - name: install rust stable
        id: rust-toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          components: llvm-tools

      - run: rustc --version --verbose

      - name: build initial wheel
        uses: PyO3/maturin-action@v1
        with:
          manylinux: auto
          args: >
            --release
            --out pgo-wheel
            --interpreter ${{ matrix.interpreter }}
          rust-toolchain: stable
          docker-options: -e CI
          working-directory: bindings
        env:
          RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"

      - name: detect rust host
        run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
        shell: bash

      - name: generate pgo data
        run: |
          pip install -U pip
          pip install -r bindings/requirements-dev.txt
          pip install powerboxes --no-index --no-deps --find-links bindings/pgo-wheel --force-reinstall
          pytest bindings/tests/test_speed.py
          rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'

      - name: merge pgo data
        run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata

      - name: build pgo-optimized wheel
        uses: PyO3/maturin-action@v1
        with:
          manylinux: auto
          args: >
            --release
            --out dist
            --interpreter ${{ matrix.interpreter }}
          rust-toolchain: stable
          docker-options: -e CI
          working-directory: bindings
        env:
          RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"

      - run: ${{ matrix.ls || 'ls -lh' }} bindings/dist/

      - uses: actions/upload-artifact@v3
        with:
          name: pypi_files_pgo
          path: bindings/dist

  inspect-pypi-assets:
    needs: [build, build-sdist, build-pgo]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: bindings/dist

      - name: list dist files before PGO builds
        run: |
         ls -lh bindings/dist/
         ls -l bindings/dist/
         echo "`ls dist | wc -l` files"

      - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds)
        uses: actions/download-artifact@v3
        with:
          name: pypi_files_pgo
          path: bindings/dist

      - name: list dist files with PGO builds
        run: |
         ls -lh bindings/dist/
         ls -l bindings/dist/
         echo "`ls dist | wc -l` files"

      - name: extract and list sdist file
        run: |
         mkdir sdist-files
         tar -xvf bindings/dist/*.tar.gz -C sdist-files
         tree -a sdist-files

      - name: extract and list wheel file
        run: |
         ls bindings/dist/*cp310-manylinux*x86_64.whl | head -n 1
         python -m zipfile --list `ls bindings/dist/*cp310-manylinux*x86_64.whl | head -n 1`

  test-builds-os:
    name: test build on ${{ matrix.os }}
    needs: [build, build-pgo]

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]

    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: bindings/dist

      - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds)
        uses: actions/download-artifact@v3
        with:
          name: pypi_files_pgo
          path: bindings/dist

      - run: pip install typing-extensions
      - run: pip install -r bindings/requirements-dev.txt
      - run: pip install powerboxes --no-index --no-deps --find-links bindings/dist --force-reinstall
      - run: pytest bindings/tests

  release:
    needs: [test-builds-os, build-sdist, check]
    if: success() && startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - run: pip install -U twine

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: bindings/dist

      - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds)
        uses: actions/download-artifact@v3
        with:
          name: pypi_files_pgo
          path: bindings/dist

      - run: twine check --strict bindings/dist/*

      - name: upload to pypi
        run: twine upload bindings/dist/*
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}