[Python][Dev] Switch to using pyproject.toml when building and running CI.
#10617
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| - '!main' | |
| - '!feature' | |
| tags: | |
| - '**' | |
| paths-ignore: | |
| - '**.md' | |
| - 'examples/**' | |
| - 'test/**' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Python.yml' | |
| pull_request: | |
| types: [opened, reopened, ready_for_review] | |
| paths-ignore: | |
| - '**.md' | |
| - 'examples/**' | |
| - 'test/**' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Python.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| jobs: | |
| # This is just a sanity check of Python 3.9 running with Arrow | |
| linux-python3-9: | |
| name: Python 3.9 Linux | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CIBW_BUILD: 'cp39-manylinux_x86_64' | |
| CIBW_TEST_COMMAND: 'python -m pytest {project}/tests --verbose' | |
| SETUPTOOLS_SCM_NO_LOCAL: 'yes' | |
| PYTEST_TIMEOUT: '600' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install | |
| shell: bash | |
| run: pip install cibuildwheel twine | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build source dist | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| python setup.py sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| - name: Build | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| manylinux-extensions-x64: | |
| # Builds extensions for linux_md64_gcc4 | |
| name: Linux Extensions (gcc4) | |
| runs-on: ubuntu-latest | |
| container: quay.io/pypa/manylinux2014_x86_64 | |
| needs: linux-python3-9 | |
| env: | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/manylinux_2014_setup | |
| with: | |
| aws-cli: 1 | |
| ninja-build: 1 | |
| ccache: 1 | |
| nodejs: 1 | |
| ssh: 1 | |
| python_alias: 1 | |
| openssl: 1 | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: x64-linux | |
| post_install: rm build/release/src/libduckdb* | |
| deploy_as: linux_amd64_gcc4 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| run_tests: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| run_autoload_tests: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| treat_warn_as_error: 0 | |
| ninja: 1 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: manylinux-extensions-x64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| linux-python3: | |
| name: Python 3 Linux | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| arch: [x86_64, aarch64] | |
| python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*] | |
| isRelease: | |
| - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} | |
| exclude: | |
| - isRelease: false | |
| python_build: 'cp37-*' | |
| - isRelease: false | |
| python_build: 'cp38-*' | |
| - isRelease: false | |
| python_build: 'cp39-*' | |
| - isRelease: false | |
| python_build: 'cp311-*' | |
| - isRelease: false | |
| arch: aarch64 | |
| needs: manylinux-extensions-x64 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| CIBW_SKIP: '*-musllinux_aarch64' | |
| CIBW_ARCHS: ${{ matrix.arch == 'aarch64' && 'aarch64' || 'auto64' }} | |
| SETUPTOOLS_SCM_NO_LOCAL: 'yes' | |
| PYTEST_TIMEOUT: '600' | |
| DUCKDB_BUILD_UNITY: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| if: ${{ matrix.arch == 'aarch64' }} | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install | |
| shell: bash | |
| run: | | |
| pip install cibuildwheel twine | |
| python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true" | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ matrix.arch == 'x86_64' }} | |
| with: | |
| name: manylinux-extensions-x64 | |
| path: tools/pythonpkg | |
| - name: List extensions to be tested | |
| shell: bash | |
| run: | | |
| find tools/pythonpkg -maxdepth 2 -type f -name "*.duckdb_extension" | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }}-${{ matrix.arch }}-${{ matrix.python_build }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cd tools/pythonpkg | |
| python setup.py sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| ls duckdb_tarball | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| ls wheelhouse | |
| - name: Deploy | |
| env: | |
| TWINE_USERNAME: '__token__' | |
| TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }} | |
| shell: bash | |
| run: | | |
| python scripts/asset-upload-gha.py duckdb_python_src.tar.gz=tools/pythonpkg/dist/duckdb-*.tar.gz | |
| if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then | |
| twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl tools/pythonpkg/dist/duckdb-*.tar.gz | |
| fi | |
| osx-python3: | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' | |
| name: Python 3 OSX | |
| runs-on: macos-latest | |
| strategy: | |
| matrix: | |
| python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*] | |
| needs: linux-python3-9 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| CIBW_ARCHS: 'x86_64 universal2 arm64' | |
| CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast --verbose' | |
| SETUPTOOLS_SCM_NO_LOCAL: 'yes' | |
| TWINE_USERNAME: '__token__' | |
| DUCKDB_BUILD_UNITY: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install | |
| shell: bash | |
| run: pip install cibuildwheel twine | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }}-${{ matrix.python_build }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cd tools/pythonpkg | |
| python setup.py sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| - name: Deploy | |
| env: | |
| TWINE_USERNAME: '__token__' | |
| TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }} | |
| shell: bash | |
| run: | | |
| if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then | |
| twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl | |
| fi | |
| win-python3: | |
| name: Python 3 Windows | |
| runs-on: windows-2019 | |
| strategy: | |
| matrix: | |
| python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*] | |
| isRelease: | |
| - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} | |
| exclude: | |
| - isRelease: false | |
| python_build: 'cp37-*' | |
| - isRelease: false | |
| python_build: 'cp38-*' | |
| - isRelease: false | |
| python_build: 'cp39-*' | |
| - isRelease: false | |
| python_build: 'cp311-*' | |
| needs: linux-python3-9 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| SETUPTOOLS_SCM_NO_LOCAL: 'yes' | |
| SETUPTOOLS_USE_DISTUTILS: 'stdlib' | |
| TWINE_USERNAME: '__token__' | |
| DUCKDB_BUILD_UNITY: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install | |
| shell: bash | |
| run: pip install cibuildwheel twine | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }}-${{ matrix.python_build }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cd tools/pythonpkg | |
| python setup.py sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| - name: Deploy | |
| env: | |
| TWINE_USERNAME: '__token__' | |
| TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }} | |
| shell: bash | |
| run: | | |
| if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then | |
| twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl | |
| fi | |
| linux-release-cleanup: | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' | |
| name: PyPi Release Cleanup | |
| runs-on: ubuntu-20.04 | |
| needs: linux-python3-9 | |
| env: | |
| PYPI_CLEANUP_USERNAME: 'mytherin' | |
| PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}} | |
| PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install Packages | |
| shell: bash | |
| run: python3 -m pip install pyotp requests | |
| - name: Cleanup Releases | |
| shell: bash | |
| run: python3 scripts/pypi_cleanup.py |