nightly-wheel.yml

name: TorchArrow Nightly Build and Test
on:
  pull_request:
    branches:
      - main
  workflow_call:
    secrets:
      AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
        required: true
      AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
        required: true

jobs:
  linux-container:
    runs-on: ubuntu-latest
    container: prestocpp/velox-sse-velox-torcharrow:kpai-20220711
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - 3.7
          - 3.8
          - 3.9
    steps:
      - name: Print CPU info
        run: cat /proc/cpuinfo

      - name: Check out source repository
        uses: actions/checkout@v2
        with:
            submodules: recursive

      - name: Build the wheel
        run: |
          source /opt/conda/etc/profile.d/conda.sh
          PYTHON_VERSION=${{ matrix.python-version }} CPU_TARGET="sse" packaging/build_wheel.sh
          conda activate env${{ matrix.python-version }}
          pip install auditwheel
          auditwheel repair dist/*.whl -w fixed_dist --plat manylinux2014_x86_64

      - name: Upload wheel to S3 Storage
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
          S3_PATH: "s3://pytorch/whl/nightly/"
        run: |
          source /opt/conda/etc/profile.d/conda.sh
          conda activate env${{ matrix.python-version }}
          pip install --user awscli
          set -x
          if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then
            echo "AWS_ACCESS_KEY_ID is not set, exiting upload"
            exit 1
          fi
          for pkg in fixed_dist/torcharrow*.whl; do
              ~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read
          done

  macos-container:
    runs-on: macos-latest
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - 3.7
          - 3.8
          - 3.9
    steps:
      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Check out source repository
        uses: actions/checkout@v2
        with:
            submodules: recursive

      - name: Build the wheel
        run: |
          MACOSX_DEPLOYMENT_TARGET=10.15 CPU_TARGET="sse" ./csrc/velox/velox/scripts/setup-macos.sh
          pip install wheel
          CPU_TARGET="sse" ./packaging/build_wheel.sh
          pip install delocate
          delocate-wheel dist/*.whl -w fixed_dist

      - name: Upload wheel to S3 Storage
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
          S3_PATH: "s3://pytorch/whl/nightly/"
        run: |
          pip install --user awscli
          set -x
          if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then
             echo "AWS_ACCESS_KEY_ID is not set, exiting upload"
             exit 1
          fi
          for pkg in fixed_dist/torcharrow*.whl; do
              ~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read
          done