cibuildwheel (platform-tag=win_amd64, python-tags=["cp38", "cp39", "cp310", "cp311", "cp312"], unoptimized=false, include-debug-info-for-macos=false, run_tests=true, use-server-rc=false, server-tag=latest) #31
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'cibuildwheel' | |
| run-name: 'cibuildwheel (platform-tag=${{ inputs.platform-tag }}, python-tags=${{ inputs.python-tags }}, unoptimized=${{ inputs.unoptimized }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }}, run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }})' | |
| # Build wheels on all (or select) Python versions supported by the Python client for a specific platform | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # Originally we were going to have 3 workflows to build a single wheel, all wheels for a platform, and all artifacts for | |
| # the Python client. But for the workflow that builds all wheels for a platform, it would have to reuse and forward | |
| # most of the same inputs from this workflow, so we would have to maintain the same inputs in two places. | |
| # Instead, combine the first two workflows into one (i.e this workflow) to help with maintainability. | |
| python-tags: | |
| type: string | |
| description: Valid JSON list of Python tags to build the client for | |
| required: false | |
| default: '["cp38", "cp39", "cp310", "cp311", "cp312"]' | |
| platform-tag: | |
| description: Platform to build the client for. | |
| type: choice | |
| required: true | |
| options: | |
| - manylinux_x86_64 | |
| - manylinux_aarch64 | |
| - macosx_x86_64 | |
| - macosx_arm64 | |
| - win_amd64 | |
| # Makes debugging via gh cli easier. | |
| default: manylinux_x86_64 | |
| # Ideally, we want to hardcode the runner OS version in this workflow to make sure the wheels are always built in the same environment | |
| # for each platform. | |
| # But it is not possible in Github Actions to create a "map" of values | |
| # so we cannot map the platform tag input to runner OS values. | |
| # | |
| # Solutions we considered: | |
| # | |
| # 1. We could create a workflow job that dynamically maps the platform tag to the runner OS, | |
| # but this would waste Github Actions resources if there were a calling workflow that runs this job multiple times | |
| # to build wheels for multiple platforms | |
| # | |
| # 2. We could create a list of string-encoded JSON lists that can be converted later into a real JSON list. | |
| # Each string encoded list is a JSON list that contains the platform tag first, and then the runner OS second. | |
| # And each encoded list is a "choice" that can be selected from a workflow_dispatch input | |
| # but we decided against this because workflow_call doesn't support "choice" inputs, so we would have to update | |
| # the runner OS everywhere that this workflow is called in code anyways. | |
| # | |
| # For now, just use two separate inputs for the platform tag and runner OS to keep things simple for both | |
| # workflow_dispatch and workflow_call events | |
| runner-os: | |
| description: Runner OS to build the client on | |
| type: choice | |
| required: true | |
| options: | |
| - ubuntu-22.04 # For both Linux x86 and ARM | |
| - macos-12-large # macOS x86 | |
| - macos-14 # macOS ARM | |
| - windows-2022 | |
| default: ubuntu-22.04 | |
| unoptimized: | |
| description: 'macOS or Linux: Apply -O0 flag?' | |
| # Windows supports a different flag to disable optimizations, but we haven't added support for it yet | |
| type: boolean | |
| required: false | |
| default: false | |
| include-debug-info-for-macos: | |
| description: 'macOS: Build wheels for debugging?' | |
| type: boolean | |
| required: false | |
| default: false | |
| run_tests: | |
| description: 'Run Aerospike server and run tests using built wheels?' | |
| type: boolean | |
| required: false | |
| default: false | |
| use-server-rc: | |
| type: boolean | |
| required: true | |
| default: false | |
| description: 'Test against server release candidate?' | |
| server-tag: | |
| required: true | |
| default: 'latest' | |
| description: 'Server docker image tag' | |
| workflow_call: | |
| inputs: | |
| # See workflow call hack in update-version.yml | |
| is_workflow_call: | |
| type: boolean | |
| default: true | |
| required: false | |
| python-tags: | |
| type: string | |
| required: true | |
| platform-tag: | |
| type: string | |
| required: true | |
| runner-os: | |
| type: string | |
| required: true | |
| # Only used in workflow_call event | |
| sha-to-build-and-test-override: | |
| type: string | |
| required: false | |
| default: ${{ github.sha }} | |
| # Debug build settings unused by calling workflows, but set here as defaults | |
| unoptimized: | |
| type: boolean | |
| required: false | |
| default: false | |
| include-debug-info-for-macos: | |
| type: boolean | |
| required: false | |
| default: false | |
| run_tests: | |
| type: boolean | |
| required: false | |
| default: false | |
| use-server-rc: | |
| required: false | |
| type: boolean | |
| default: false | |
| description: 'Test against server release candidate?' | |
| server-tag: | |
| required: false | |
| type: string | |
| default: 'latest' | |
| description: 'Server docker image tag' | |
| jobs: | |
| cibuildwheel: | |
| strategy: | |
| matrix: | |
| python-tag: ${{ fromJSON(inputs.python-tags) }} | |
| fail-fast: false | |
| runs-on: ${{ inputs.runner-os }} | |
| steps: | |
| - name: Get build identifier for cibuildwheel | |
| run: echo "BUILD_IDENTIFIER=${{ matrix.python-tag }}-${{ inputs.platform-tag }}" >> $GITHUB_ENV | |
| # Powershell (the default shell for Windows) has a different syntax for setting environment variables | |
| shell: bash | |
| - name: Get commit sha to show status check | |
| # The workflow_dispatch event doesn't have the sha-to-build-and-test-override input | |
| # But set-commit-status-action requires a valid github sha | |
| run: echo COMMIT_SHA_TO_BUILD_AND_TEST=${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test-override || github.sha }} >> $GITHUB_ENV | |
| shell: bash | |
| - name: Create status check message | |
| run: echo STATUS_CHECK_MESSAGE="cibuildwheel (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Show job status for commit | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} | |
| context: ${{ env.STATUS_CHECK_MESSAGE }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} | |
| # We need the last tag before the ref, so we can relabel the version if needed | |
| fetch-depth: 0 | |
| - name: 'macOS arm64: Install experimental Python 3.8 macOS arm64 build' | |
| # By default, cibuildwheel installs and uses Python 3.8 x86_64 to cross compile macOS arm64 wheels | |
| # There is a bug that builds macOS x86_64 wheels instead, so we install this Python 3.8 native ARM build to ensure | |
| # the wheel is compiled for macOS arm64 | |
| # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 | |
| if: ${{ matrix.python-tag == 'cp38' && inputs.platform-tag == 'macosx_arm64' }} | |
| run: | | |
| curl -o /tmp/Python38.pkg https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg | |
| sudo installer -pkg /tmp/Python38.pkg -target / | |
| sh "/Applications/Python 3.8/Install Certificates.command" | |
| - name: 'Windows: Add msbuild to PATH' | |
| if: ${{ inputs.platform-tag == 'win_amd64' }} | |
| uses: microsoft/setup-msbuild@v1.1 | |
| - name: 'Windows: Install C client deps' | |
| if: ${{ inputs.platform-tag == 'win_amd64' }} | |
| run: nuget restore | |
| working-directory: aerospike-client-c/vs | |
| - name: 'Linux aarch64: Set up QEMU for building and testing wheels' | |
| if: ${{ startsWith(inputs.platform-tag, 'manylinux') && endsWith(inputs.platform-tag, 'aarch64') }} | |
| uses: docker/setup-qemu-action@v2 | |
| with: | |
| platforms: arm64 | |
| # Note that environment variables in Github are all strings | |
| # Github mac m1 and windows runners don't support Docker / nested virtualization | |
| # so we need to use self-hosted runners for those platforms | |
| - run: echo "RUN_INTEGRATION_TESTS_IN_THIS_JOB=${{ inputs.run_tests && (startsWith(inputs.platform-tag, 'manylinux') || inputs.platform-tag == 'macosx_x86_64') }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: 'macOS x86: Setup Docker using colima for testing' | |
| if: ${{ env.RUN_INTEGRATION_TESTS_IN_THIS_JOB == 'true' && inputs.platform-tag == 'macosx_x86_64' }} | |
| uses: ./.github/actions/setup-docker-on-macos | |
| - name: 'macOS x86: run Aerospike server in Docker container and connect via localhost' | |
| uses: ./.github/actions/run-ee-server | |
| if: ${{ env.RUN_INTEGRATION_TESTS_IN_THIS_JOB == 'true' && inputs.platform-tag == 'macosx_x86_64' }} | |
| - name: "Linux: run Aerospike server in Docker container and configure config.conf to connect to the server container's Docker IP address" | |
| uses: ./.github/actions/run-ee-server-for-ext-container | |
| if: ${{ env.RUN_INTEGRATION_TESTS_IN_THIS_JOB == 'true' && startsWith(inputs.platform-tag, 'manylinux') }} | |
| - name: If not running tests against server, only run basic import test | |
| if: ${{ env.RUN_INTEGRATION_TESTS_IN_THIS_JOB == 'false' }} | |
| # Use double quotes otherwise Windows will throw this error in cibuildwheel | |
| # 'import | |
| # ^ | |
| # SyntaxError: EOL while scanning string literal | |
| run: echo "TEST_COMMAND=python -c \"import aerospike\"" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Otherwise, enable integration tests | |
| if: ${{ env.RUN_INTEGRATION_TESTS_IN_THIS_JOB == 'true' }} | |
| run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Set unoptimize flag | |
| if: ${{ inputs.unoptimized && (startsWith(inputs.platform-tag, 'manylinux') || startsWith(inputs.platform-tag, 'macosx')) }} | |
| run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV | |
| - name: Set include dsym flag | |
| if: ${{ inputs.include-debug-info-for-macos && startsWith(inputs.platform-tag, 'macosx') }} | |
| run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV | |
| - name: Build wheel | |
| uses: pypa/cibuildwheel@v2.18.1 | |
| env: | |
| CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.unoptimized && 'UNOPTIMIZED' || '' }} | |
| CIBW_ENVIRONMENT_MACOS: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 | |
| # We are cross compiling for Linux aarch64 on x86_64 hardware, so we need to specify the target arch | |
| CIBW_ARCHS_LINUX: ${{ inputs.platform-tag == 'manylinux_aarch64' && 'aarch64' || 'auto' }} | |
| CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }} | |
| CIBW_BUILD_FRONTEND: build | |
| CIBW_BEFORE_ALL_LINUX: > | |
| yum install openssl-devel -y && | |
| yum install python-devel -y && | |
| yum install python-setuptools -y | |
| CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" | |
| CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" | |
| CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} | |
| - name: Upload wheels to GitHub | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| path: ./wheelhouse/*.whl | |
| name: ${{ env.BUILD_IDENTIFIER }}.build | |
| - name: Set final commit status | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} | |
| status: ${{ job.status }} | |
| context: ${{ env.STATUS_CHECK_MESSAGE }} | |
| test-self-hosted: | |
| needs: cibuildwheel | |
| if: ${{ inputs.run_tests && (inputs.platform-tag == 'macosx_arm64' || inputs.platform-tag == 'win_amd64') }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-tag: ${{ fromJSON(inputs.python-tags) }} | |
| runs-on: ${{ inputs.platform-tag == 'macosx_arm64' && fromJSON('["self-hosted", "macOS", "ARM64"]') || fromJSON('["self-hosted", "Windows", "X64"]') }} | |
| steps: | |
| # TODO: copy and pasted status check code from cibuildwheel job, but need to refactor later | |
| - name: Get build identifier for cibuildwheel | |
| run: echo "BUILD_IDENTIFIER=${{ matrix.python-tag }}-${{ inputs.platform-tag }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Get commit sha to show status check | |
| run: echo COMMIT_SHA_TO_BUILD_AND_TEST=${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test-override || github.sha }} >> $GITHUB_ENV | |
| shell: bash | |
| - name: Create status check message | |
| run: echo STATUS_CHECK_MESSAGE="Test on self hosted (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Show job status for commit | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| with: | |
| sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} | |
| context: ${{ env.STATUS_CHECK_MESSAGE }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} | |
| # Need to be able to save Docker Hub credentials to keychain | |
| - if: ${{ inputs.platform-tag == 'macosx_arm64' && inputs.use-server-rc }} | |
| run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} | |
| - if: ${{ inputs.use-server-rc }} | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} | |
| password: ${{ secrets.DOCKER_HUB_BOT_PW }} | |
| - name: Convert Python tag to Python version | |
| # Don't use sed because we want this command to work on both mac and windows | |
| # The command used in GNU sed is different than in macOS sed | |
| run: | | |
| PYTHON_TAG=${{ matrix.python-tag }} | |
| PYTHON_VERSION="${PYTHON_TAG/cp/}" | |
| echo PYTHON_VERSION="${PYTHON_VERSION/3/3.}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Set default Python version we're testing via PATH env var | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - uses: ./.github/actions/run-ee-server | |
| - name: Create config.conf | |
| run: cp config.conf.template config.conf | |
| working-directory: test | |
| shell: bash | |
| - name: Download wheel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.BUILD_IDENTIFIER }}.build | |
| - name: Install wheel | |
| run: python3 -m pip install aerospike --force-reinstall --no-index --find-links=./ | |
| shell: bash | |
| - name: Connect to Docker container on remote machine with Docker daemon | |
| if: ${{ inputs.platform-tag == 'win_amd64' }} | |
| # DOCKER_HOST contains the IP address of the remote machine | |
| run: | | |
| python3 -m pip install crudini==0.9.4 | |
| $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")} | |
| python3 -m crudini --set config.conf community-edition hosts ${env:DOCKER_HOST_IP}:3000 | |
| working-directory: test | |
| - run: python3 -m pip install pytest -c requirements.txt | |
| working-directory: test | |
| shell: bash | |
| - run: python3 -m pytest new_tests/ | |
| working-directory: test | |
| shell: bash | |
| - name: Show job status for commit | |
| if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} | |
| uses: myrotvorets/set-commit-status-action@v2.0.0 | |
| with: | |
| sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} | |
| status: ${{ job.status }} | |
| context: ${{ env.STATUS_CHECK_MESSAGE }} |