From 51974f4c4a22486fcd7f1532b0a927c50c12b6ca Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Mon, 28 Feb 2022 12:17:13 -0800 Subject: [PATCH 1/6] Add nightly workflow to build torcharrow. --- .github/workflows/velox.yml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/velox.yml diff --git a/.github/workflows/velox.yml b/.github/workflows/velox.yml new file mode 100644 index 000000000..c137d575d --- /dev/null +++ b/.github/workflows/velox.yml @@ -0,0 +1,54 @@ +name: Velox Nightly Build and Test +on: + schedule: + - cron: 0 0 * * * + +jobs: + vm: + runs-on: ubuntu-latest + + container: + runs-on: ubuntu-latest + container: prestocpp/velox-torcharrow:kpai-20220228 + steps: + - name: Print CPU info + run: cat /proc/cpuinfo + + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Check out source repository + uses: actions/checkout@v2 + + - name: Update submodules + run: git submodule update --init --recursive + + # Based on https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + run: | + echo "::set-output name=timestamp::$(/bin/date -u "+%Y%m%d-%H:%M:%S")" + shell: bash + + - name: Load ccache files + uses: actions/cache@v2 + with: + path: .ccache + key: velox-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + restore-keys: | + velox-ccache- + + - name: Bulid TorchArrow + run: | + CCACHE_DIR=$GITHUB_WORKSPACE/.ccache ccache -sz + CCACHE_DIR=$GITHUB_WORKSPACE/.ccache python setup.py develop --user + + - name: Install dependencies for test + run: | + pip3 install pytest --user + pip3 install torch==1.10.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html --user + + - name : Run TorchArrow unit tests + run: pytest --no-header -v torcharrow/test From afaf8a75dc72a24ca18f08f925eb40ff405aa7d2 Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Thu, 31 Mar 2022 14:24:45 -0700 Subject: [PATCH 2/6] Add support for macos build --- .github/workflows/linux-nightly.yml | 51 +++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux-nightly.yml b/.github/workflows/linux-nightly.yml index c247ea78f..7a5745fdb 100644 --- a/.github/workflows/linux-nightly.yml +++ b/.github/workflows/linux-nightly.yml @@ -1,7 +1,8 @@ name: TorchArrow Nightly Build and Test on: - schedule: - - cron: "0 0 * * *" + push: + branches: + - main workflow_call: secrets: AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: @@ -10,7 +11,7 @@ on: required: true jobs: - container: + linux-container: runs-on: ubuntu-latest container: prestocpp/velox-velox-torcharrow:kpai-20220301 strategy: @@ -54,3 +55,47 @@ jobs: for pkg in fixed_dist/torcharrow*.whl; do ~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read done + + macos-container: + runs-on: macos-latest + strategy: + fail-fast: true + matrix: + python-version: + - 3.7 + - 3.8 + - 3.9 + steps: + - name: Print CPU info + run: cat /proc/cpuinfo + + - name: Check out source repository + uses: actions/checkout@v2 + with: + submodules: true + + - name: Build the wheel + run: | + source /opt/conda/etc/profile.d/conda.sh + PYTHON_VERSION=${{ matrix.python-version }} packaging/build_wheel.sh + conda activate env${{ matrix.python-version }} + pip install delocate + delocate-wheel dist/*.whl -w fixed_dist + + - name: Upload wheel to S3 Storage + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} + S3_PATH: "s3://pytorch/whl/nightly/" + run: | + source /opt/conda/etc/profile.d/conda.sh + conda activate env${{ matrix.python-version }} + pip install --user awscli + set -x + if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then + echo "AWS_ACCESS_KEY_ID is not set, exiting upload" + exit 1 + fi + for pkg in fixed_dist/torcharrow*.whl; do + ~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read + done From 72988d3c4d6531f9f4340dd21967b36eb50e0cda Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Fri, 3 Jun 2022 14:16:31 -0700 Subject: [PATCH 3/6] Fix SSE build. --- csrc/velox/velox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csrc/velox/velox b/csrc/velox/velox index bb62f5511..d7e4058a5 160000 --- a/csrc/velox/velox +++ b/csrc/velox/velox @@ -1 +1 @@ -Subproject commit bb62f5511efa1d34668424e483448b94b1904ce8 +Subproject commit d7e4058a59255cc16d852d39bd927e779c63cfc4 From 56f5169107218c9546e20ed41e991732f1f635cd Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Wed, 15 Jun 2022 13:10:12 -0700 Subject: [PATCH 4/6] Add new image with bison/flex as dependencies. --- .github/workflows/nightly-wheel.yml | 7 ++-- .github/workflows/velox.yml | 54 ----------------------------- 2 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/velox.yml diff --git a/.github/workflows/nightly-wheel.yml b/.github/workflows/nightly-wheel.yml index cea918054..1172e927a 100644 --- a/.github/workflows/nightly-wheel.yml +++ b/.github/workflows/nightly-wheel.yml @@ -1,7 +1,8 @@ name: TorchArrow Nightly Build and Test on: - schedule: - - cron: "0 0 * * *" + pull_request: + branches: + - main workflow_call: secrets: AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: @@ -12,7 +13,7 @@ on: jobs: linux-container: runs-on: ubuntu-latest - container: prestocpp/velox-sse-velox-torcharrow:kpai-20220524 + container: prestocpp/velox-sse-velox-torcharrow:kpai-20220615 strategy: fail-fast: false matrix: diff --git a/.github/workflows/velox.yml b/.github/workflows/velox.yml deleted file mode 100644 index c137d575d..000000000 --- a/.github/workflows/velox.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Velox Nightly Build and Test -on: - schedule: - - cron: 0 0 * * * - -jobs: - vm: - runs-on: ubuntu-latest - - container: - runs-on: ubuntu-latest - container: prestocpp/velox-torcharrow:kpai-20220228 - steps: - - name: Print CPU info - run: cat /proc/cpuinfo - - - name: Setup Python environment - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Check out source repository - uses: actions/checkout@v2 - - - name: Update submodules - run: git submodule update --init --recursive - - # Based on https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - run: | - echo "::set-output name=timestamp::$(/bin/date -u "+%Y%m%d-%H:%M:%S")" - shell: bash - - - name: Load ccache files - uses: actions/cache@v2 - with: - path: .ccache - key: velox-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} - restore-keys: | - velox-ccache- - - - name: Bulid TorchArrow - run: | - CCACHE_DIR=$GITHUB_WORKSPACE/.ccache ccache -sz - CCACHE_DIR=$GITHUB_WORKSPACE/.ccache python setup.py develop --user - - - name: Install dependencies for test - run: | - pip3 install pytest --user - pip3 install torch==1.10.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html --user - - - name : Run TorchArrow unit tests - run: pytest --no-header -v torcharrow/test From 9d2b3bf1bdb5c27d6fab048cb191fe04ae79d032 Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Wed, 15 Jun 2022 14:00:07 -0700 Subject: [PATCH 5/6] Update to working sha --- csrc/velox/velox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csrc/velox/velox b/csrc/velox/velox index f0ef3012b..9a0f5a1a7 160000 --- a/csrc/velox/velox +++ b/csrc/velox/velox @@ -1 +1 @@ -Subproject commit f0ef3012b426c2a2cbb8824940a9d655518e62b5 +Subproject commit 9a0f5a1a74cce09a52909b33beff1e7166cfe8cf From c91d7645a910fca2aed3e6fde7db158d614f5a85 Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Wed, 15 Jun 2022 18:35:17 -0700 Subject: [PATCH 6/6] Switch to latest velox sha with fix --- .github/workflows/nightly-wheel.yml | 5 ++--- csrc/velox/velox | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-wheel.yml b/.github/workflows/nightly-wheel.yml index 1172e927a..9e46143aa 100644 --- a/.github/workflows/nightly-wheel.yml +++ b/.github/workflows/nightly-wheel.yml @@ -1,8 +1,7 @@ name: TorchArrow Nightly Build and Test on: - pull_request: - branches: - - main + schedule: + - cron: "0 0 * * *" workflow_call: secrets: AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: diff --git a/csrc/velox/velox b/csrc/velox/velox index 9a0f5a1a7..1f53c8d1f 160000 --- a/csrc/velox/velox +++ b/csrc/velox/velox @@ -1 +1 @@ -Subproject commit 9a0f5a1a74cce09a52909b33beff1e7166cfe8cf +Subproject commit 1f53c8d1fd7ea559f174385a36f61a6ceb1a243a