Build wheels (run_tests=true, use-server-rc=false, server-tag=latest, test-macos-x86=false, apply-no-optimizations=false, include-debug-info-for-macos=false) #811
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build wheels | |
| run-name: Build wheels (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}, test-macos-x86=${{ inputs.test-macos-x86 }}, apply-no-optimizations=${{ inputs.apply-no-optimizations }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }}) | |
| # Builds manylinux wheels and source distribution | |
| # If running tests, publish results in commit status | |
| # For each build, upload to Github as an artifact if it passes testing or does not need to run tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # If we only want to check that the builds pass on an arbitrary branch | |
| run_tests: | |
| description: "Run integration tests" | |
| required: true | |
| type: boolean | |
| default: false | |
| use-server-rc: | |
| type: boolean | |
| required: true | |
| default: false | |
| description: 'Test against server release candidate?' | |
| # If we are creating a backport and want to test an arbitrary branch against an older server version | |
| server-tag: | |
| required: true | |
| default: 'latest' | |
| description: 'Server docker image tag' | |
| test-macos-x86: | |
| required: true | |
| type: boolean | |
| default: false | |
| description: 'Test macOS x86 wheels (unstable)' | |
| apply-no-optimizations: | |
| required: true | |
| type: boolean | |
| default: false | |
| description: 'Linux and macOS: apply -O0 when building C and Python client?' | |
| include-debug-info-for-macos: | |
| required: true | |
| type: boolean | |
| default: false | |
| description: 'macOS: include source files and line numbers for both C and Python client for debugging?' | |
| workflow_call: | |
| inputs: | |
| # The "dev" tests test the artifacts against a server release | |
| # The "stage" tests and release workflow only need to build the artifacts, not test them | |
| run_tests: | |
| description: "Run integration tests" | |
| required: false | |
| type: boolean | |
| default: false | |
| ref: | |
| type: string | |
| required: false | |
| # Calling workflow doesn't actually use the options below | |
| # But we need to set default values for workflow calls to use | |
| use-server-rc: | |
| required: false | |
| default: true | |
| type: boolean | |
| server-tag: | |
| type: string | |
| required: false | |
| default: 'latest' | |
| test-macos-x86: | |
| required: false | |
| type: boolean | |
| default: false | |
| apply-no-optimizations: | |
| required: false | |
| type: boolean | |
| default: false | |
| include-debug-info-for-macos: | |
| required: false | |
| type: boolean | |
| default: false | |
| secrets: | |
| DOCKER_HUB_BOT_USERNAME: | |
| required: false | |
| DOCKER_HUB_BOT_PW: | |
| required: false | |
| MAC_M1_SELF_HOSTED_RUNNER_PW: | |
| required: false | |
| jobs: | |
| build-sdist: | |
| name: Build source distribution | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Show job status for commit | |
| # If workflow_call triggered this job, github.event_name will inherit the event of the calling workflow | |
| # https://github.com/actions/runner/issues/3146#issuecomment-2000017097 | |
| # Commit status will already be shown by the calling workflow for these events | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| with: | |
| sha: ${{ github.sha }} | |
| context: "Build wheels (sdist)" | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| - name: Install build dependencies (pip packages) | |
| run: python3 -m pip install -r requirements.txt | |
| - name: Build source distribution | |
| run: python3 -m build --sdist | |
| - name: Upload source distribution to GitHub | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./dist/*.tar.gz | |
| name: sdist.build | |
| - name: Set final commit status | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| # Always run even if job failed or is cancelled | |
| # But we don't want to show anything if the calling workflow was triggered by these events | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| status: ${{ job.status }} | |
| context: "Build wheels (sdist)" | |
| manylinux: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Python versions to build wheels on | |
| python: [ | |
| "cp38", | |
| "cp39", | |
| "cp310", | |
| "cp311", | |
| "cp312" | |
| ] | |
| platform: [ | |
| ["x86_64", "ubuntu-22.04"], | |
| ["aarch64", "aerospike_arm_runners_2"] | |
| ] | |
| runs-on: ${{ matrix.platform[1] }} | |
| steps: | |
| - name: Show job status for commit | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})" | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| - name: Set up QEMU for cross compiling arm64 | |
| if: ${{ matrix.platform[0] == 'aarch64' }} | |
| uses: docker/setup-qemu-action@v2 | |
| with: | |
| platforms: arm64 | |
| - uses: ./.github/actions/run-ee-server-for-ext-container | |
| if: ${{ inputs.run_tests }} | |
| with: | |
| use-server-rc: ${{ inputs.use-server-rc }} | |
| server-tag: ${{ inputs.server-tag }} | |
| docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} | |
| docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} | |
| - name: Enable tests | |
| if: ${{ inputs.run_tests }} | |
| run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV | |
| - name: Disable tests (only run basic import test) | |
| if: ${{ !inputs.run_tests }} | |
| run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV | |
| - name: Set unoptimize flag | |
| if: ${{ inputs.apply-no-optimizations }} | |
| run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV | |
| - name: Build wheel | |
| uses: pypa/cibuildwheel@v2.19.2 | |
| env: | |
| CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.apply-no-optimizations && 'UNOPTIMIZED' || '' }} | |
| CIBW_BUILD: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }} | |
| CIBW_BUILD_FRONTEND: build | |
| CIBW_BEFORE_ALL_LINUX: > | |
| yum install openssl-devel -y && | |
| yum install python-devel -y && | |
| yum install python-setuptools -y | |
| CIBW_ARCHS: "${{ matrix.platform[0] }}" | |
| CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} | |
| - name: Upload wheels to GitHub | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| path: ./wheelhouse/*.whl | |
| name: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }}.build | |
| - name: Set final commit status | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| status: ${{ job.status }} | |
| context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})" | |
| macOS-x86: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: [ | |
| "cp38", | |
| "cp39", | |
| "cp310", | |
| "cp311", | |
| "cp312" | |
| ] | |
| runs-on: macos-12-large | |
| steps: | |
| - name: Show job status for commit | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| context: "Build wheels (${{ matrix.python }}-macosx_x86_64)" | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| - name: Install Docker Engine | |
| if: ${{ inputs.run_tests }} | |
| run: brew install colima | |
| - name: Install Docker client | |
| if: ${{ inputs.run_tests }} | |
| run: brew install docker | |
| - name: Start Docker Engine | |
| if: ${{ inputs.run_tests }} | |
| run: colima start | |
| - uses: ./.github/actions/run-ee-server | |
| if: ${{ inputs.run_tests }} | |
| with: | |
| use-server-rc: ${{ inputs.use-server-rc }} | |
| server-tag: ${{ inputs.server-tag }} | |
| docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} | |
| docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} | |
| - name: Enable tests | |
| if: ${{ inputs.run_tests && inputs.test-macos-x86 }} | |
| run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV | |
| - name: Disable tests (only run basic import test) | |
| if: ${{ !inputs.run_tests || !inputs.test-macos-x86 }} | |
| run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV | |
| - name: Set unoptimize flag | |
| if: ${{ inputs.apply-no-optimizations }} | |
| run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV | |
| - name: Set include dsym flag | |
| if: ${{ inputs.include-debug-info-for-macos }} | |
| run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV | |
| - name: Build wheel | |
| uses: pypa/cibuildwheel@v2.19.2 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python }}-macosx_x86_64 | |
| CIBW_BUILD_FRONTEND: build | |
| CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 | |
| CIBW_ARCHS: "x86_64" | |
| CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} | |
| - name: Save macOS wheel | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: ${{ matrix.python }}-macosx_x86_64.build | |
| path: wheelhouse/*.whl | |
| - name: Set final commit status | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| status: ${{ job.status }} | |
| sha: ${{ github.sha }} | |
| context: "Build wheels (${{ matrix.python }}-macosx_x86_64)" | |
| macOS-m1: | |
| runs-on: [ | |
| self-hosted, | |
| macOS, | |
| ARM64 | |
| ] | |
| strategy: | |
| matrix: | |
| python-version: [ | |
| ["cp38", "3.8"], | |
| ["cp39", "3.9"], | |
| ["cp310", "3.10"], | |
| ["cp311", "3.11"], | |
| ["cp312", "3.12"] | |
| ] | |
| fail-fast: false | |
| steps: | |
| - name: Show job status for commit | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)" | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| # Update dependencies if needed | |
| - name: Add brew to path | |
| run: echo PATH=$PATH:/opt/homebrew/bin/ >> $GITHUB_ENV | |
| - name: Install or upgrade Python | |
| run: brew install python@${{ matrix.python-version[1] }} | |
| - name: Install or upgrade OpenSSL 1.1 | |
| run: brew install openssl@1.1 | |
| - name: Set environment variables for building | |
| run: | | |
| openssl_path=$(brew --prefix openssl@1.1) | |
| echo SSL_LIB_PATH="$openssl_path/lib/" >> $GITHUB_ENV | |
| echo CPATH="$openssl_path/include/" >> $GITHUB_ENV | |
| echo STATIC_SSL=1 >> $GITHUB_ENV | |
| - name: Install pip build packages | |
| run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt | |
| # Self-hosted runner only | |
| # Need to be able to save Docker Hub credentials to keychain | |
| - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} | |
| if: ${{ inputs.run_tests && inputs.use-server-rc }} | |
| - if: ${{ inputs.run_tests && inputs.use-server-rc }} | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} | |
| password: ${{ secrets.DOCKER_HUB_BOT_PW }} | |
| - name: Use server rc | |
| if: ${{ inputs.run_tests && inputs.use-server-rc }} | |
| run: echo IMAGE_NAME="${{ vars.SERVER_RC_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV | |
| - name: Use server release | |
| if: ${{ inputs.run_tests && !inputs.use-server-rc }} | |
| run: echo IMAGE_NAME="${{ vars.SERVER_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV | |
| - name: Run server | |
| if: ${{ inputs.run_tests }} | |
| run: docker run -d -p 3000:3000 --pull always --name aerospike ${{ env.IMAGE_NAME }} | |
| - name: Set unoptimize flag | |
| if: ${{ inputs.apply-no-optimizations }} | |
| run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV | |
| - name: Set include dsym flag | |
| if: ${{ inputs.include-debug-info-for-macos }} | |
| run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV | |
| - run: python${{ matrix.python-version[1] }} -m build | |
| - name: Install delocate | |
| run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall delocate -c ./requirements.txt | |
| working-directory: .github/workflows | |
| - name: Create config.conf | |
| run: cp config.conf.template config.conf | |
| working-directory: test | |
| - run: delocate-wheel --require-archs "arm64" -w wheelhouse/ -v dist/*.whl | |
| - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --find-links=wheelhouse/ --no-index --force-reinstall aerospike | |
| - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt | |
| if: ${{ inputs.run_tests }} | |
| working-directory: test | |
| - run: python${{ matrix.python-version[1] }} -m pytest new_tests/ | |
| if: ${{ inputs.run_tests }} | |
| working-directory: test | |
| - run: python${{ matrix.python-version[1] }} -c "import aerospike" | |
| if: ${{ !inputs.run_tests }} | |
| - name: Save macOS wheel | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: ${{ matrix.python-version[0] }}-macosx_arm64.build | |
| path: wheelhouse/*.whl | |
| - name: Stop server | |
| if: ${{ always() && inputs.run_tests }} | |
| run: | | |
| docker container stop aerospike | |
| docker container prune -f | |
| - name: Set final commit status | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| status: ${{ job.status }} | |
| context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)" | |
| windows-build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: [ | |
| ["cp38", "3.8"], | |
| ["cp39", "3.9"], | |
| ["cp310", "3.10"], | |
| ["cp311", "3.11"], | |
| ["cp312", "3.12"] | |
| ] | |
| runs-on: windows-2022 | |
| steps: | |
| - name: Show job status for commit | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| context: "Build wheels (${{ matrix.python[0] }}-win_amd64)" | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| - name: Add msbuild to PATH | |
| uses: microsoft/setup-msbuild@v1.1 | |
| - name: Install C client deps | |
| run: nuget restore | |
| working-directory: aerospike-client-c/vs | |
| - name: Build wheel | |
| uses: pypa/cibuildwheel@v2.19.2 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python[0] }}-win_amd64 | |
| CIBW_BUILD_FRONTEND: build | |
| CIBW_ARCHS: auto64 | |
| CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" | |
| CIBW_REPAIR_WHEEL_COMMAND: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./wheelhouse/*.whl | |
| name: ${{ matrix.python[0] }}-win_amd64.build | |
| - name: Set final commit status | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| status: ${{ job.status }} | |
| context: "Build wheels (${{ matrix.python[0] }}-win_amd64)" | |
| test-windows: | |
| needs: windows-build | |
| if: ${{ inputs.run_tests }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: [ | |
| ["cp38", "3.8"], | |
| ["cp39", "3.9"], | |
| ["cp310", "3.10"], | |
| ["cp311", "3.11"], | |
| ["cp312", "3.12"] | |
| ] | |
| runs-on: [self-hosted, Windows, X64] | |
| steps: | |
| - name: Show job status for commit | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ github.sha }} | |
| context: "Test Windows (${{ matrix.python[0] }}-win_amd64)" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.ref }} | |
| - run: docker run -d -p 3000:3000 --name aerospike aerospike/aerospike-server | |
| - name: Download wheel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.python[0] }}-win_amd64.build | |
| - name: Create config.conf | |
| run: cp config.conf.template config.conf | |
| working-directory: test | |
| - name: Install wheel | |
| run: python${{ matrix.python[1] }} -m pip install aerospike --force-reinstall --no-index --find-links=./ | |
| - name: Connect to Docker container on remote machine with Docker daemon | |
| # DOCKER_HOST contains the IP address of the remote machine | |
| run: | | |
| python${{ matrix.python[1] }} -m pip install crudini==0.9.4 | |
| $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")} | |
| python${{ matrix.python[1] }} -m crudini --set config.conf community-edition hosts ${env:DOCKER_HOST_IP}:3000 | |
| working-directory: test | |
| - run: python${{ matrix.python[1] }} -m pip install pytest -c requirements.txt | |
| working-directory: test | |
| - run: python${{ matrix.python[1] }} -m pytest new_tests/ | |
| working-directory: test | |
| - name: Cleanup | |
| if: ${{ always() }} | |
| run: | | |
| docker stop aerospike | |
| docker container rm aerospike | |
| - name: Show job status for commit | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| with: | |
| sha: ${{ github.sha }} | |
| status: ${{ job.status }} | |
| context: "Test Windows (${{ matrix.python[0] }}-win_amd64)" |