build-wheels.yml

name: Build wheels
run-name: Build wheels (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}, test-macos-x86=${{ inputs.test-macos-x86 }}, apply-no-optimizations=${{ inputs.apply-no-optimizations }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }})

# Builds manylinux wheels and source distribution
# If running tests, publish results in commit status
# For each build, upload to Github as an artifact if it passes testing or does not need to run tests

on:
  workflow_dispatch:
    inputs:
      # If we only want to check that the builds pass on an arbitrary branch
      run_tests:
        description: "Run integration tests"
        required: true
        type: boolean
        default: false
      use-server-rc:
        type: boolean
        required: true
        default: false
        description: 'Test against server release candidate?'
      # If we are creating a backport and want to test an arbitrary branch against an older server version
      server-tag:
        required: true
        default: 'latest'
        description: 'Server docker image tag'
      test-macos-x86:
        required: true
        type: boolean
        default: false
        description: 'Test macOS x86 wheels (unstable)'
      apply-no-optimizations:
        required: true
        type: boolean
        default: false
        description: 'Linux and macOS: apply -O0 when building C and Python client?'
      include-debug-info-for-macos:
        required: true
        type: boolean
        default: false
        description: 'macOS: include source files and line numbers for both C and Python client for debugging?'

  workflow_call:
    inputs:
      # The "dev" tests test the artifacts against a server release
      # The "stage" tests and release workflow only need to build the artifacts, not test them
      run_tests:
        description: "Run integration tests"
        required: false
        type: boolean
        default: false
      ref:
        type: string
        required: false
      # Calling workflow doesn't actually use the options below
      # But we need to set default values for workflow calls to use
      use-server-rc:
        required: false
        default: true
        type: boolean
      server-tag:
        type: string
        required: false
        default: 'latest'
      test-macos-x86:
        required: false
        type: boolean
        default: false
      apply-no-optimizations:
        required: false
        type: boolean
        default: false
      include-debug-info-for-macos:
        required: false
        type: boolean
        default: false
    secrets:
      DOCKER_HUB_BOT_USERNAME:
        required: false
      DOCKER_HUB_BOT_PW:
        required: false
      MAC_M1_SELF_HOSTED_RUNNER_PW:
        required: false

jobs:
  build-sdist:
    name: Build source distribution
    runs-on: ubuntu-22.04
    steps:
    - name: Show job status for commit
      # If workflow_call triggered this job, github.event_name will inherit the event of the calling workflow
      # https://github.com/actions/runner/issues/3146#issuecomment-2000017097
      # Commit status will already be shown by the calling workflow for these events
      if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }}
      uses: myrotvorets/set-commit-status-action@v2.0.0
      with:
        sha: ${{ github.sha }}
        context: "Build wheels (sdist)"

    - uses: actions/checkout@v4
      with:
        submodules: recursive
        ref: ${{ inputs.ref }}
        fetch-depth: 0

    - name: Install build dependencies (pip packages)
      run: python3 -m pip install -r requirements.txt

    - name: Build source distribution
      run: python3 -m build --sdist

    - name: Upload source distribution to GitHub
      uses: actions/upload-artifact@v4
      with:
        path: ./dist/*.tar.gz
        name: sdist.build

    - name: Set final commit status
      uses: myrotvorets/set-commit-status-action@v2.0.0
      # Always run even if job failed or is cancelled
      # But we don't want to show anything if the calling workflow was triggered by these events
      if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }}
      with:
        sha: ${{ github.sha }}
        status: ${{ job.status }}
        context: "Build wheels (sdist)"

  manylinux:
    strategy:
      fail-fast: false
      matrix:
        # Python versions to build wheels on
        python: [
          "cp38",
          "cp39",
          "cp310",
          "cp311",
          "cp312"
        ]
        platform: [
          ["x86_64", "ubuntu-22.04"],
          ["aarch64", "aerospike_arm_runners_2"]
        ]
    runs-on: ${{ matrix.platform[1] }}
    steps:
    - name: Show job status for commit
      uses: myrotvorets/set-commit-status-action@v2.0.0
      if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }}
      with:
        sha: ${{ github.sha }}
        context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})"

    - uses: actions/checkout@v4
      with:
        submodules: recursive
        ref: ${{ inputs.ref }}
        fetch-depth: 0

    - uses: ./.github/actions/run-ee-server-for-ext-container
      if: ${{ inputs.run_tests }}
      with:
        use-server-rc: ${{ inputs.use-server-rc }}
        server-tag: ${{ inputs.server-tag }}
        docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
        docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}

    - name: Enable tests
      if: ${{ inputs.run_tests }}
      run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV

    - name: Disable tests (only run basic import test)
      if: ${{ !inputs.run_tests }}
      run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV

    - name: Set unoptimize flag
      if: ${{ inputs.apply-no-optimizations }}
      run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV

    - name: Build wheel
      uses: pypa/cibuildwheel@v2.19.2
      env:
        CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.apply-no-optimizations && 'UNOPTIMIZED' || '' }}
        CIBW_BUILD: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }}
        CIBW_BUILD_FRONTEND: build
        CIBW_BEFORE_ALL_LINUX: >
          yum install openssl-devel -y &&
          yum install python-devel -y &&
          yum install python-setuptools -y
        CIBW_ARCHS: "${{ matrix.platform[0] }}"
        CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }}

    - name: Upload wheels to GitHub
      uses: actions/upload-artifact@v4
      if: ${{ always() }}
      with:
        path: ./wheelhouse/*.whl
        name: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }}.build

    - name: Set final commit status
      uses: myrotvorets/set-commit-status-action@v2.0.0
      if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }}
      with:
        sha: ${{ github.sha }}
        status: ${{ job.status }}
        context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})"

  macOS-x86:
    strategy:
      fail-fast: false
      matrix:
        python: [
          "cp38",
          "cp39",
          "cp310",
          "cp311",
          "cp312"
        ]
    runs-on: macos-12-large
    steps:
    - name: Show job status for commit
      uses: myrotvorets/set-commit-status-action@v2.0.0
      if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }}
      with:
        sha: ${{ github.sha }}
        context: "Build wheels (${{ matrix.python }}-macosx_x86_64)"

    - uses: actions/checkout@v4
      with:
        submodules: recursive
        ref: ${{ inputs.ref }}
        fetch-depth: 0

    - name: Install Docker Engine
      if: ${{ inputs.run_tests }}
      run: brew install colima

    - name: Install Docker client
      if: ${{ inputs.run_tests }}
      run: brew install docker

    - name: Start Docker Engine
      if: ${{ inputs.run_tests }}
      run: colima start

    - uses: ./.github/actions/run-ee-server
      if: ${{ inputs.run_tests }}
      with:
        use-server-rc: ${{ inputs.use-server-rc }}
        server-tag: ${{ inputs.server-tag }}
        docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
        docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}

    - name: Enable tests
      if: ${{ inputs.run_tests && inputs.test-macos-x86 }}
      run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV

    - name: Disable tests (only run basic import test)
      if: ${{ !inputs.run_tests || !inputs.test-macos-x86 }}
      run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV

    - name: Set unoptimize flag
      if: ${{ inputs.apply-no-optimizations }}
      run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV

    - name: Set include dsym flag
      if: ${{ inputs.include-debug-info-for-macos }}
      run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV

    - name: Build wheel
      uses: pypa/cibuildwheel@v2.19.2
      env:
        CIBW_BUILD: ${{ matrix.python }}-macosx_x86_64
        CIBW_BUILD_FRONTEND: build
        CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1
        CIBW_ARCHS: "x86_64"
        CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }}

    - name: Save macOS wheel
      uses: actions/upload-artifact@v4
      if: ${{ always() }}
      with:
        name: ${{ matrix.python }}-macosx_x86_64.build
        path: wheelhouse/*.whl

    - name: Set final commit status
      uses: myrotvorets/set-commit-status-action@v2.0.0
      if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }}
      with:
        status: ${{ job.status }}
        sha: ${{ github.sha }}
        context: "Build wheels (${{ matrix.python }}-macosx_x86_64)"

  macOS-m1:
    runs-on: [
      self-hosted,
      macOS,
      ARM64
    ]
    strategy:
      matrix:
        python-version: [
          ["cp38", "3.8"],
          ["cp39", "3.9"],
          ["cp310", "3.10"],
          ["cp311", "3.11"],
          ["cp312", "3.12"]
        ]
      fail-fast: false
    steps:
    - name: Show job status for commit
      uses: myrotvorets/set-commit-status-action@v2.0.0
      if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }}
      with:
        sha: ${{ github.sha }}
        context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)"

    - uses: actions/checkout@v4
      with:
        submodules: recursive
        ref: ${{ inputs.ref }}
        fetch-depth: 0

    # Update dependencies if needed
    - name: Add brew to path
      run: echo PATH=$PATH:/opt/homebrew/bin/ >> $GITHUB_ENV

    - name: Install or upgrade Python
      run: brew install python@${{ matrix.python-version[1] }}

    - name: Install or upgrade OpenSSL 1.1
      run: brew install openssl@1.1

    - name: Set environment variables for building
      run: |
        openssl_path=$(brew --prefix openssl@1.1)
        echo SSL_LIB_PATH="$openssl_path/lib/" >> $GITHUB_ENV
        echo CPATH="$openssl_path/include/" >> $GITHUB_ENV
        echo STATIC_SSL=1 >> $GITHUB_ENV

    - name: Install pip build packages
      run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt

    # Self-hosted runner only
    # Need to be able to save Docker Hub credentials to keychain
    - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }}
      if: ${{ inputs.run_tests && inputs.use-server-rc }}

    - if: ${{ inputs.run_tests && inputs.use-server-rc }}
      uses: docker/login-action@v3
      with:
        username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
        password: ${{ secrets.DOCKER_HUB_BOT_PW }}

    - name: Use server rc
      if: ${{ inputs.run_tests && inputs.use-server-rc }}
      run: echo IMAGE_NAME="${{ vars.SERVER_RC_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV

    - name: Use server release
      if: ${{ inputs.run_tests && !inputs.use-server-rc }}
      run: echo IMAGE_NAME="${{ vars.SERVER_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV

    - name: Run server
      if: ${{ inputs.run_tests }}
      run: docker run -d -p 3000:3000 --pull always --name aerospike ${{ env.IMAGE_NAME }}

    - name: Set unoptimize flag
      if: ${{ inputs.apply-no-optimizations }}
      run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV

    - name: Set include dsym flag
      if: ${{ inputs.include-debug-info-for-macos }}
      run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV

    - run: python${{ matrix.python-version[1] }} -m build

    - name: Install delocate
      run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall delocate -c ./requirements.txt
      working-directory: .github/workflows

    - name: Create config.conf
      run: cp config.conf.template config.conf
      working-directory: test

    - run: delocate-wheel --require-archs "arm64" -w wheelhouse/ -v dist/*.whl
    - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --find-links=wheelhouse/ --no-index --force-reinstall aerospike

    - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt
      if: ${{ inputs.run_tests }}
      working-directory: test

    - run: python${{ matrix.python-version[1] }} -m pytest new_tests/
      if: ${{ inputs.run_tests }}
      working-directory: test

    - run: python${{ matrix.python-version[1] }} -c "import aerospike"
      if: ${{ !inputs.run_tests }}

    - name: Save macOS wheel
      uses: actions/upload-artifact@v4
      if: ${{ always() }}
      with:
        name: ${{ matrix.python-version[0] }}-macosx_arm64.build
        path: wheelhouse/*.whl

    - name: Stop server
      if: ${{ always() && inputs.run_tests }}
      run: |
        docker container stop aerospike
        docker container prune -f

    - name: Set final commit status
      uses: myrotvorets/set-commit-status-action@v2.0.0
      if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }}
      with:
        sha: ${{ github.sha }}
        status: ${{ job.status }}
        context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)"

  windows-build:
    strategy:
      fail-fast: false
      matrix:
        python: [
          ["cp38", "3.8"],
          ["cp39", "3.9"],
          ["cp310", "3.10"],
          ["cp311", "3.11"],
          ["cp312", "3.12"]
        ]
    runs-on: windows-2022
    steps:
      - name: Show job status for commit
        uses: myrotvorets/set-commit-status-action@v2.0.0
        if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }}
        with:
          sha: ${{ github.sha }}
          context: "Build wheels (${{ matrix.python[0] }}-win_amd64)"

      - uses: actions/checkout@v4
        with:
          submodules: recursive
          ref: ${{ inputs.ref }}
          fetch-depth: 0

      - name: Add msbuild to PATH
        uses: microsoft/setup-msbuild@v1.1

      - name: Install C client deps
        run: nuget restore
        working-directory: aerospike-client-c/vs

      - name: Build wheel
        uses: pypa/cibuildwheel@v2.19.2
        env:
          CIBW_BUILD: ${{ matrix.python[0] }}-win_amd64
          CIBW_BUILD_FRONTEND: build
          CIBW_ARCHS: auto64
          CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel"
          CIBW_REPAIR_WHEEL_COMMAND: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}"

      - uses: actions/upload-artifact@v4
        with:
          path: ./wheelhouse/*.whl
          name: ${{ matrix.python[0] }}-win_amd64.build

      - name: Set final commit status
        uses: myrotvorets/set-commit-status-action@v2.0.0
        if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }}
        with:
          sha: ${{ github.sha }}
          status: ${{ job.status }}
          context: "Build wheels (${{ matrix.python[0] }}-win_amd64)"

  test-windows:
    needs: windows-build
    if: ${{ inputs.run_tests }}
    strategy:
      fail-fast: false
      matrix:
        python: [
          ["cp38", "3.8"],
          ["cp39", "3.9"],
          ["cp310", "3.10"],
          ["cp311", "3.11"],
          ["cp312", "3.12"]
        ]
    runs-on: [self-hosted, Windows, X64]
    steps:
      - name: Show job status for commit
        uses: myrotvorets/set-commit-status-action@v2.0.0
        if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }}
        with:
          sha: ${{ github.sha }}
          context: "Test Windows (${{ matrix.python[0] }}-win_amd64)"
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}

      - run: docker run -d -p 3000:3000 --name aerospike aerospike/aerospike-server
      - name: Download wheel
        uses: actions/download-artifact@v4
        with:
          name: ${{ matrix.python[0] }}-win_amd64.build

      - name: Create config.conf
        run: cp config.conf.template config.conf
        working-directory: test

      - name: Install wheel
        run: python${{ matrix.python[1] }} -m pip install aerospike --force-reinstall --no-index --find-links=./
      - name: Connect to Docker container on remote machine with Docker daemon
        # DOCKER_HOST contains the IP address of the remote machine
        run: |
          python${{ matrix.python[1] }} -m pip install crudini==0.9.4
          $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")}
          python${{ matrix.python[1] }} -m crudini --set config.conf community-edition hosts ${env:DOCKER_HOST_IP}:3000
        working-directory: test

      - run: python${{ matrix.python[1] }} -m pip install pytest -c requirements.txt
        working-directory: test

      - run: python${{ matrix.python[1] }} -m pytest new_tests/
        working-directory: test

      - name: Cleanup
        if: ${{ always() }}
        run: |
          docker stop aerospike
          docker container rm aerospike

      - name: Show job status for commit
        if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }}
        uses: myrotvorets/set-commit-status-action@v2.0.0
        with:
          sha: ${{ github.sha }}
          status: ${{ job.status }}
          context: "Test Windows (${{ matrix.python[0] }}-win_amd64)"