Use cmake for third party

pytorch · Jun 30, 2020 · 75c62ac · 75c62ac
1 parent 4daf2fb
commit 75c62ac
Show file tree

Hide file tree

Showing 10 changed files with 167 additions and 400 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -38,6 +38,17 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
+  install_cmake_macos:
+    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+    steps:
+      - run:
+          name: Install cmake
+          command: |
+            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
+            mkdir cmake
+            tar -xf cmake.tar.gz --strip 3 -C cmake
+            rm cmake.tar.gz
+            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
 
 binary_common: &binary_common
   parameters:
@@ -82,28 +93,36 @@ jobs:
 
   download_third_parties_nix:
     docker:
-      - image: "centos:8"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: small
     steps:
       - checkout
       - generate_cache_key
       - restore_cache:
 
           keys:
-            - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+            - tp-nix-v1-{{ checksum ".cachekey" }}
 
       - run:
-          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+          command: |
+              mkdir -p third_party/archives/
+              wget --directory-prefix=third_party/archives/ \
+                  https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz \
+                  https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz \
+                  https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz \
+                  https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz \
+                  https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz \
+                  https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
       - save_cache:
 
-          key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          key: tp-nix-v1-{{ checksum ".cachekey" }}
 
           paths:
-            - third_party/tmp
+            - third_party/archives
       - persist_to_workspace:
           root: third_party
           paths:
-            - tmp
+            - archives
 
   binary_linux_wheel:
     <<: *binary_common
@@ -145,6 +164,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -169,6 +189,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -383,19 +404,21 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +446,21 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +567,8 @@ jobs:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh

diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -38,6 +38,17 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
+  install_cmake_macos:
+    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+    steps:
+      - run:
+          name: Install cmake
+          command: |
+            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
+            mkdir cmake
+            tar -xf cmake.tar.gz --strip 3 -C cmake
+            rm cmake.tar.gz
+            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
 
 binary_common: &binary_common
   parameters:
@@ -82,28 +93,36 @@ jobs:
 
   download_third_parties_nix:
     docker:
-      - image: "centos:8"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: small
     steps:
       - checkout
       - generate_cache_key
       - restore_cache:
           {% raw %}
           keys:
-            - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+            - tp-nix-v1-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
-          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+          command: |
+              mkdir -p third_party/archives/
+              wget --directory-prefix=third_party/archives/ \
+                  https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz \
+                  https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz \
+                  https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz \
+                  https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz \
+                  https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz \
+                  https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
       - save_cache:
           {% raw %}
-          key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          key: tp-nix-v1-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
-            - third_party/tmp
+            - third_party/archives
       - persist_to_workspace:
           root: third_party
           paths:
-            - tmp
+            - archives
 
   binary_linux_wheel:
     <<: *binary_common
@@ -145,6 +164,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -169,6 +189,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -383,19 +404,21 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +446,21 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +567,8 @@ jobs:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh

diff --git a/.circleci/unittest/linux/docker/Dockerfile b/.circleci/unittest/linux/docker/Dockerfile
@@ -58,6 +58,8 @@ RUN apt update && apt install -y \
         sox \
         libsox-dev \
         libsox-fmt-all \
+        cmake \
+        ninja-build \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /kaldi /kaldi
 COPY --from=builder /third_party /third_party

diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh
@@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
 python -m torch.utils.collect_env
-export PATH="${PWD}/third_party/build/bin/:${PATH}"
+export PATH="${PWD}/third_party/install/bin/:${PATH}"
 pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test
diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh
@@ -33,5 +33,10 @@ conda activate "${env_dir}"
 printf "* Installing dependencies (except PyTorch)\n"
 conda env update --file "${this_dir}/environment.yml" --prune
 
-# 4. Build codecs
-build_tools/setup_helpers/build_third_party.sh
+# 4. Buld codecs
+mkdir -p third_party/build
+(
+    cd third_party/build
+    cmake ..
+    cmake --build .
+)
diff --git a/.gitignore b/.gitignore
@@ -121,3 +121,8 @@ gen.yml
 examples/interactive_asr/data/*.txt
 examples/interactive_asr/data/*.model
 examples/interactive_asr/data/*.pt
+
+# third parties
+third_party/archives/
+third_party/install/
+third_party/src/
diff --git a/build_tools/setup_helpers/build_third_party.sh b/build_tools/setup_helpers/build_third_party.sh