Add binary build jobs (#766)

pytorch · May 13, 2020 · 3bdf25a · 3bdf25a
1 parent ab3629a
commit 3bdf25a
Show file tree

Hide file tree

Showing 12 changed files with 619 additions and 38 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -103,10 +103,13 @@ jobs:
     steps:
       - checkout
       - run:
-          # Cannot easily deduplicate this as source'ing activate
-          # will set environment variables which we need to propagate
-          # to build_wheel.sh
+          # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution.
           command: |
+            curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz
+            tar -xzf cmake.tar.gz
+            cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/
+            cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/
+            export PATH="${PATH}:/usr/local/bin"
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
@@ -316,13 +319,48 @@ jobs:
           command: .circleci/unittest/scripts/run_style_checks.sh
 
 workflows:
-  # build:
-  #   jobs:
-  #     - circleci_consistency
-  #     
-  #     - binary_win_conda:
-  #         name: torchtext_win_py3.6
-  #         python_version: "3.6"
+  build:
+    jobs:
+      - circleci_consistency
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py3.6
+          python_version: '3.6'
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py3.7
+          python_version: '3.7'
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py3.8
+          python_version: '3.8'
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py3.6
+          python_version: '3.6'
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py3.7
+          python_version: '3.7'
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py3.8
+          python_version: '3.8'
+      - binary_linux_conda:
+          name: binary_linux_conda_py3.6
+          python_version: '3.6'
+      - binary_linux_conda:
+          name: binary_linux_conda_py3.7
+          python_version: '3.7'
+      - binary_linux_conda:
+          name: binary_linux_conda_py3.8
+          python_version: '3.8'
+      - binary_macos_conda:
+          name: binary_macos_conda_py3.6
+          python_version: '3.6'
+      - binary_macos_conda:
+          name: binary_macos_conda_py3.7
+          python_version: '3.7'
+      - binary_macos_conda:
+          name: binary_macos_conda_py3.8
+          python_version: '3.8'
+      #  - binary_win_conda:
+      #     name: torchtext_win_py3.6
+      #     python_version: "3.6"
   unittest:
     jobs:
       - unittest_linux:
@@ -343,7 +381,222 @@ workflows:
           filters:
             branches:
               only: nightly
-      # 
+      - binary_linux_wheel:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.6
+          python_version: '3.6'
+      - binary_wheel_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.6_upload
+          requires:
+          - nightly_binary_linux_wheel_py3.6
+      - smoke_test_linux_pip:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.6_smoke_test_pip
+          python_version: '3.6'
+          requires:
+          - nightly_binary_linux_wheel_py3.6_upload
+      - binary_linux_wheel:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.7
+          python_version: '3.7'
+      - binary_wheel_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.7_upload
+          requires:
+          - nightly_binary_linux_wheel_py3.7
+      - smoke_test_linux_pip:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.7_smoke_test_pip
+          python_version: '3.7'
+          requires:
+          - nightly_binary_linux_wheel_py3.7_upload
+      - binary_linux_wheel:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.8
+          python_version: '3.8'
+      - binary_wheel_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.8_upload
+          requires:
+          - nightly_binary_linux_wheel_py3.8
+      - smoke_test_linux_pip:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_wheel_py3.8_smoke_test_pip
+          python_version: '3.8'
+          requires:
+          - nightly_binary_linux_wheel_py3.8_upload
+      - binary_macos_wheel:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_wheel_py3.6
+          python_version: '3.6'
+      - binary_wheel_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_wheel_py3.6_upload
+          requires:
+          - nightly_binary_macos_wheel_py3.6
+      - binary_macos_wheel:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_wheel_py3.7
+          python_version: '3.7'
+      - binary_wheel_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_wheel_py3.7_upload
+          requires:
+          - nightly_binary_macos_wheel_py3.7
+      - binary_macos_wheel:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_wheel_py3.8
+          python_version: '3.8'
+      - binary_wheel_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_wheel_py3.8_upload
+          requires:
+          - nightly_binary_macos_wheel_py3.8
+      - binary_linux_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.6
+          python_version: '3.6'
+      - binary_conda_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.6_upload
+          requires:
+          - nightly_binary_linux_conda_py3.6
+      - smoke_test_linux_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.6_smoke_test_conda
+          python_version: '3.6'
+          requires:
+          - nightly_binary_linux_conda_py3.6_upload
+      - binary_linux_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.7
+          python_version: '3.7'
+      - binary_conda_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.7_upload
+          requires:
+          - nightly_binary_linux_conda_py3.7
+      - smoke_test_linux_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.7_smoke_test_conda
+          python_version: '3.7'
+          requires:
+          - nightly_binary_linux_conda_py3.7_upload
+      - binary_linux_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.8
+          python_version: '3.8'
+      - binary_conda_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.8_upload
+          requires:
+          - nightly_binary_linux_conda_py3.8
+      - smoke_test_linux_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_linux_conda_py3.8_smoke_test_conda
+          python_version: '3.8'
+          requires:
+          - nightly_binary_linux_conda_py3.8_upload
+      - binary_macos_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_conda_py3.6
+          python_version: '3.6'
+      - binary_conda_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_conda_py3.6_upload
+          requires:
+          - nightly_binary_macos_conda_py3.6
+      - binary_macos_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_conda_py3.7
+          python_version: '3.7'
+      - binary_conda_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_conda_py3.7_upload
+          requires:
+          - nightly_binary_macos_conda_py3.7
+      - binary_macos_conda:
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_conda_py3.8
+          python_version: '3.8'
+      - binary_conda_upload:
+          context: org-member
+          filters:
+            branches:
+              only: nightly
+          name: nightly_binary_macos_conda_py3.8_upload
+          requires:
+          - nightly_binary_macos_conda_py3.8
   docker_build:
     triggers:
       - schedule:

diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -103,10 +103,13 @@ jobs:
     steps:
       - checkout
       - run:
-          # Cannot easily deduplicate this as source'ing activate
-          # will set environment variables which we need to propagate
-          # to build_wheel.sh
+          # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution.
           command: |
+            curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz
+            tar -xzf cmake.tar.gz
+            cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/
+            cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/
+            export PATH="${PATH}:/usr/local/bin"
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
@@ -316,13 +319,13 @@ jobs:
           command: .circleci/unittest/scripts/run_style_checks.sh
 
 workflows:
-  # build:
-  #   jobs:
-  #     - circleci_consistency
-  #     {{ build_workflows() }}
-  #     - binary_win_conda:
-  #         name: torchtext_win_py3.6
-  #         python_version: "3.6"
+  build:
+    jobs:
+      - circleci_consistency
+      {{ build_workflows() }}
+      #  - binary_win_conda:
+      #     name: torchtext_win_py3.6
+      #     python_version: "3.6"
   unittest:
     jobs:
       {{ unittest_workflows() }}
@@ -332,7 +335,7 @@ workflows:
           filters:
             branches:
               only: nightly
-      # {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
+      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
   docker_build:
     triggers:
       - schedule:

diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py
@@ -23,16 +23,12 @@
 
 
 def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
-    '''
     w = []
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos"]:
             for python_version in PYTHON_VERSIONS:
                 w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload)
-
     return indent(indentation, w)
-    '''
-    return ''
 
 
 def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):

diff --git a/.gitignore b/.gitignore
@@ -120,3 +120,5 @@ venv.bak/
 # vim
 *.swp
 *.swo
+
+torchtext/version.py
diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. "$script_dir/pkg_helpers.bash"
+
+export BUILD_TYPE="conda"
+export NO_CUDA_PACKAGE=1
+setup_env 0.6.0
+export SOURCE_ROOT_DIR="$PWD"
+setup_conda_pytorch_constraint
+conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchtext
diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. "$script_dir/pkg_helpers.bash"
+
+export BUILD_TYPE="wheel"
+export NO_CUDA_PACKAGE=1
+setup_env 0.6.0
+setup_wheel_python
+pip_install numpy future
+setup_pip_pytorch_version
+git submodule update --init --recursive
+python setup.py clean
+python setup.py bdist_wheel