Conda nightly build to Github Actions (#319)

Summary:
In this PR I'm added a new Github workflow to enable nightly build for conda. But right now for developing and testing, I left the trigger as new PR comes in, so it's not really "nightly build" yet.

Multiple points to note:
* Since Github Action machines mac don't support AVX instructions, I left the test blank for mac builds, and only enabled it for linux build.
* For some reason, glog needs to be on 0.4.0 version (latest 0..5.0) to make the testing part work (even it's just a simple import), otherwise some symbol not found error will arise. I think it has something to do with glog itself, so I'm leaving the workaround as is for now.
* I actually tried to run the unit tests with linux build, and it mostly worked except for two that need to check package names. I think when running tests from conda the top level folder is just `torcharrow`, so the failing tests will have messages like "torcharrow.test" != "test". I'll leave this to further investigation and depending on whether we really want to run unittests in conda build workflows, we may end up not bothering with it.

This PR finishes the building part, but I still need to upload the built artifact to conda, which I'll leave as a todo.

Pull Request resolved: #319

Reviewed By: wenleix, ejguan

Differential Revision: D36254423

Pulled By: bearzx

fbshipit-source-id: f16a5ecf325745d57b125fe89618c203ba31d0cf

.github/workflows/nightly-conda.yml

@@ -0,0 +1,113 @@
+name: TorchArrow Nightly Build and Test for Conda
+on:
+#  schedule:
+#    - cron: "0 0 * * *"
+  pull_request:
+    branches:
+      - main
+      # For PR created by ghstack
+      - gh/*/*/base
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  linux-container:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - 3.7
+          - 3.8
+          - 3.9
+    steps:
+      - name: Print CPU info
+        run: |
+          cat /proc/cpuinfo
+
+      - name: Create Conda Env
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: conda_build_env
+
+      - name: Check out source repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      # Based on https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/
+      - name: Prepare ccache timestamp
+        id: ccache_cache_timestamp
+        run: |
+          echo "::set-output name=timestamp::$(/bin/date -u "+%Y%m%d-%H:%M:%S")"
+        shell: bash
+
+      - name: Load ccache files
+        uses: actions/cache@v2
+        with:
+          path: .ccache
+          key: ubuntu-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
+          restore-keys: |
+              ubuntu-ccache-
+
+      - name: Install dependencies with APT
+        run: |
+          sudo apt-get update
+          sudo apt install -y g++ cmake ccache ninja-build checkinstall git \
+          libssl-dev libboost-all-dev libdouble-conversion-dev libgoogle-glog-dev \
+          libgflags-dev libevent-dev libre2-dev
+
+      # Based on https://github.com/facebookincubator/velox/blob/99429407c3d524e07b32b8b19a03aa7382f819cf/.circleci/config.yml#L114-L116
+      - name: Configure ccache
+        run: |
+          echo "$GITHUB_WORKSPACE"
+          CCACHE_DIR=$GITHUB_WORKSPACE/.ccache_root ccache -sz -M 1G
+          CCACHE_DIR=$GITHUB_WORKSPACE/.ccache ccache -sz -M 1G
+
+      - name: Build and install folly and fmt
+        # sudo doesn't preserve environment vairable; set it after sudo: https://stackoverflow.com/questions/8633461/how-to-keep-environment-variables-when-using-sudo/33183620#33183620
+        run: |
+          sudo CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache CCACHE_DIR=$GITHUB_WORKSPACE/.ccache_root scripts/setup-ubuntu.sh
+
+      - name: Conda Build
+        shell: bash -l {0}
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        run: |
+          # source /opt/conda/etc/profile.d/conda.sh
+          conda activate conda_build_env
+          conda install -yq conda-build -c conda-forge
+          ./packaging/build_conda.sh
+
+  macos-container:
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - 3.7
+          - 3.8
+          - 3.9
+    steps:
+      - name: Create Conda Env
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: conda_build_env
+
+      - name: Check out source repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Conda Build
+        shell: bash -l {0}
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        run: |
+          conda activate conda_build_env
+          conda install -yq conda-build -c conda-forge
+          MACOSX_DEPLOYMENT_TARGET=10.15 ./csrc/velox/velox/scripts/setup-macos.sh
+          ./packaging/build_conda.sh

packaging/torcharrow/meta.yaml

@@ -7,53 +7,41 @@ source:
 
 requirements:
   build:
-    # - { { compiler('c') } } # [win]
-    # - { { compiler('cxx') } } # [win]
     - cmake
     - python
-    {% if osx %}
-    - boost
-    {% elif linux %}
-    - ninja-build
-    - checkinstall
-    - libssl-dev
-    - libboost-all-dev
-    - libdouble-conversion-dev
-    - libgoogle-glog-dev
-    - libgflags-dev
-    - libevent-dev
-    - libre2-dev
-    {% endif %}
   host:
     - python
     - setuptools
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }}
   run:
     - python
     - numpy
-    - boost
-    - libboost
     - typing_inspect
     - pandas
     - typing
     - tabulate
     - pyarrow
     - arrow
     - cffi
+    - glog==0.4.0
     {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
 
 build:
   string: py{{py}}
   script_env:
     - BUILD_VERSION
 
+#GitHub Actions usually provide Mac without AVX2 support, for now just skip the test until Velox can be run without AVX2
+#pytest --no-header -v torcharrow/test
 test:
+  {% if linux %}
   imports:
     - torcharrow
   source_files:
     - torcharrow/test
   requires:
     - pytest
+  {% endif %}
 
 about:
   home: https://github.com/facebookresearch/torcharrow