Merge pull request #624 from chrishkchris/SINGA-487_conda

SINGA-487 Include NCCL and MPICH in conda build

src/io/communicator.cc

@@ -22,7 +22,7 @@
 
 #ifdef USE_DIST
 
-#include "./math_kernel.h"
+#include "../core/tensor/math_kernel.h"
 #include "singa/io/communicator.h"
 
 namespace singa {

tool/conda/cpu/README.md

@@ -17,26 +17,26 @@
     under the License.
 -->
 The conda package specification includes the package name (i.e. singa), version and build string (could be very long).
-To install a certain Singa package, we run
+To install a certain SINGA package, we run
 
     conda install -c nusdbsystem singa=<version>=<build string>
 
 It is inconvenient to type all 3 parts when running the installation commands.
 The meta.yml file in this folder is to create a conda package `singa-cpu` as
-an alias of one specific Singa package.
+an alias of one specific SINGA package.
 It does nothing except creating a dummy conda package that depends on one real
-cpu version singa package.  For example, the following line in meta.yml indicates
-that singa-cpu depends on singa with version 1.1.1, python version=3.6
+cpu version SINGA package.  For example, the following line in meta.yml indicates
+that singa-cpu depends on SINGA with version 1.1.1, python version=3.6
 
     - singa 1.1.1 py36_cpu
 
 Therefore, when we run
 
     conda install -c nusdbsystem singa-cpu
 
-The dependent Singa package will be installed.
-By default, singa-cpu depends on the latest Singa (py3.6).
-When we have a new Singa version available, we need to update the meta.yml file to
+The dependent SINGA package will be installed.
+By default, singa-cpu depends on the latest SINGA (py3.6).
+When we have a new SINGA version available, we need to update the meta.yml file to
 change the dependency.
 
 To build this package and upload it

tool/conda/cpu/meta.yaml

@@ -17,15 +17,13 @@
 # under the License.
 #
 
-{% set version = "2.1.0.dev" %}
-
 package:
   name: singa-cpu
-  version: {{ version }}
+  version: {{ environ.get('GIT_DESCRIBE_TAG') }}
 
 requirements:
   run:
-    - singa {{ version }} cpu_py{{ py }}
+    - singa {{ environ.get('GIT_DESCRIBE_TAG') }} cpu_py{{ py }}
 
 build:
   number: 0

tool/conda/dist/README.md

@@ -0,0 +1,51 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+The conda package specification includes the package name (i.e. singa), version and build string (could be very long).
+To install a certain SINGA package we run
+
+    conda install -c nusdbsystem singa=<version>=<build string>
+
+It is inconvenient to type all 3 parts when running the installation commands.
+The meta.yml file in this folder is to create a conda package `singa-dist` as
+an alias of one specific SINGA package.
+It does nothing except creating a dummy conda package that depends on one real
+gpu version SINGA package.  For example, the following line in meta.yml indicates
+that singa-gpu depends on SINGA with version 1.1.1, python version=3.6, cuda version=9,
+cudnn version = 7.1.2, nccl version = 2.4.8.1, and mpich version = 3.3.2
+
+    - singa 1.1.1 py36_cuda9.0_cudnn7.1.2_nccl2.4.8.1_mpich3.3.2
+
+
+Therefore, when we run
+
+    conda install -c nusdbsystem singa-dist
+
+The dependent SINGA package will be installed.
+By default, singa-dist depends on the latest SINGA (py3.6) on the latest cuda (and cudnn),
+as well as the distributed computing libraries nccl and mpich.
+When we have a new SINGA version available, we need to update the meta.yml file to
+change the dependency.
+
+To build this package and upload it
+
+    conda config --add channels nusdbsystem
+    conda build .
+    anaconda -t $ANACONDA_UPLOAD_TOKEN upload -u nusdbsystem -l main <path to the singa-dist package>
+
+where $ANACONDA_UPLOAD_TOKEN is the upload token associated with nusdbsystem account on anaconda cloud.

tool/conda/dist/meta.yaml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+package:
+  name: singa-dist
+  version: {{ environ.get('GIT_DESCRIBE_TAG') }}
+
+requirements:
+  run:
+    - singa {{ environ.get('GIT_DESCRIBE_TAG') }} cudnn7.3.1_cuda10.0_nccl2.4.8.1_mpich3.3.2_py{{ py }}
+
+build:
+  number: 0
+  string: py{{ py }}
+
+
+about:
+  home: http://singa.apache.org/
+  license: Apache V2
+  summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers

tool/conda/gpu/README.md

@@ -17,16 +17,16 @@
     under the License.
 -->
 The conda package specification includes the package name (i.e. singa), version and build string (could be very long).
-To install a certain Singa package we run
+To install a certain SINGA package we run
 
     conda install -c nusdbsystem singa=<version>=<build string>
 
 It is inconvenient to type all 3 parts when running the installation commands.
 The meta.yml file in this folder is to create a conda package `singa-gpu` as
-an alias of one specific Singa package.
+an alias of one specific SINGA package.
 It does nothing except creating a dummy conda package that depends on one real
-gpu version singa package.  For example, the following line in meta.yml indicates
-that singa-gpu depends on singa with version 1.1.1, python version=3.6, cuda version=9
+gpu version SINGA package.  For example, the following line in meta.yml indicates
+that singa-gpu depends on SINGA with version 1.1.1, python version=3.6, cuda version=9
 and cudnn version = 7.1.2
 
     - singa 1.1.1 py36_cuda9.0_cudnn7.1.2
@@ -36,9 +36,9 @@ Therefore, when we run
 
     conda install -c nusdbsystem singa-gpu
 
-The dependent Singa package will be installed.
-By default, singa-gpu depends on the latest Singa (py3.6) on the latest cuda (and cudnn).
-When we have a new Singa version available, we need to update the meta.yml file to
+The dependent SINGA package will be installed.
+By default, singa-gpu depends on the latest SINGA (py3.6) on the latest cuda (and cudnn).
+When we have a new SINGA version available, we need to update the meta.yml file to
 change the dependency.
 
 To build this package and upload it

tool/conda/gpu/meta.yaml

@@ -17,15 +17,13 @@
 # under the License.
 #
 
-{% set version = "2.1.0.dev" %}
-
 package:
   name: singa-gpu
-  version: {{ version }}
+  version: {{ environ.get('GIT_DESCRIBE_TAG') }}
 
 requirements:
   run:
-    - singa {{ version }} cudnn7.3.1_cuda10.0_py{{ py }}
+    - singa {{ environ.get('GIT_DESCRIBE_TAG') }} cudnn7.3.1_cuda10.0_py{{ py }}
 
 build:
   number: 0

tool/conda/singa/README.md

@@ -16,23 +16,33 @@
     specific language governing permissions and limitations
     under the License.
 -->
-# Package Singa using conda-build
+# Package SINGA using conda-build
 
 [conda-build](https://conda.io/docs/user-guide/tasks/build-packages/index.html) is a packaging tool like apt-get, which is associated with [anaconda cloud](https://anaconda.org/) for package management for both python and cpp libraries.
 
 
 ## Environment variables
 
-We export the CUDA version if Singa is compiled with CUDA enabled. The cuDNN version is fixed by Singa and cuDNN is installed from [anaconda cloud](https://anaconda.org/anaconda/cudnn).
+We export the CUDA version if SINGA is compiled with CUDA enabled. The cuDNN version is fixed by SINGA and cuDNN is installed from [anaconda cloud](https://anaconda.org/anaconda/cudnn).
 
-    # for singa with gpu, e.g. cuda9.0-cudnn7.3.1
+    # for SINGA with GPU, e.g. cuda9.0-cudnn7.3.1
     export CUDA=9.0
 
-For CPU-only version, we do not export CUDA.
+Then, we export a flag DIST to indicate if SINGA is compiled with distributed training enabled.
+
+    # to enable distributed training: DIST=ON, otherwise: DIST=OFF
+    export DIST=OFF
+
+We need to export both CUDA and DIST for GPU version. For CPU-only version, we do not export CUDA and DIST.
 
 ## Instruction
 
-After exporting the environment variables, execute the following command to compile Singa and package it
+After exporting the environment variables, we need to add the necessary conda channels
+
+    conda config --add channels conda-forge
+    conda config --add channels nusdbsystem
+
+Then, we can execute the following commands to compile SINGA and package it
 
     conda-build .  --python 3.6
 

tool/conda/singa/build.sh

@@ -31,9 +31,17 @@ else
 	USE_CUDA=ON
 fi
 
+
+if [ $DIST == "ON" ]; then
+	USE_DIST=ON
+else
+	USE_DIST=OFF
+fi
+
 mkdir build
 cd build
 cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA \
-	-DUSE_PYTHON3=ON -DUSE_DNNL=ON -DCMAKE_OSX_SYSROOT=${CONDA_BUILD_SYSROOT} ..
+	-DUSE_PYTHON3=ON -DUSE_DNNL=ON -DUSE_DIST=$USE_DIST -DCMAKE_OSX_SYSROOT=${CONDA_BUILD_SYSROOT} ..
+
 make
 make install

tool/conda/singa/conda_build_config.yaml

@@ -32,7 +32,13 @@ dnnl:
 python:
     - 3.6
     - 3.7
+nccl:
+    - 2.4.8.1
+mpich:
+    - 3.3.2
 build_str:
-    - "cudnn7.3.1_cuda10.0"   # [environ.get("CUDA")=="10.0"]
-    - "cudnn7.3.1_cuda9.0"    # [environ.get("CUDA")=="9.0"]
+    - "cudnn7.3.1_cuda10.0"   # [environ.get("CUDA")=="10.0"] && [environ.get("DIST")=="OFF"]
+    - "cudnn7.3.1_cuda9.0"    # [environ.get("CUDA")=="9.0"] && [environ.get("DIST")=="OFF"]
     - "cpu"              # [environ.get("CUDA", "")== ""]
+    - "cudnn7.3.1_cuda10.0_nccl2.4.8.1_mpich3.3.2"     # [environ.get("CUDA")=="10.0"] && [environ.get("DIST")=="ON"]
+    - "cudnn7.3.1_cuda9.0_nccl2.4.8.1_mpich3.3.2"     # [environ.get("CUDA")=="9.0"] && [environ.get("DIST")=="ON"]

tool/conda/singa/meta.yaml

@@ -34,6 +34,7 @@ build:
   number: 0
   script_env:
     - CUDA   # run `export CUDA=9.0` in the terminal
+    - DIST   # run `export DIST=ON` in the terminal
   string: {{ build_str }}_py{{ py }}
 
 requirements:
@@ -45,29 +46,32 @@ requirements:
 
   host:
     - swig 3.0.12
-    - openblas 0.3.3
-    - protobuf 3.6.1
+    - openblas 0.3.9
+    - protobuf 3.9.2
     - glog 0.3.5
-    - numpy 1.16.0
+    - numpy 1.16.5
     - cudnn {{ cudnn }}       # ['cudnn' in str(build_str)]
     - dnnl {{ dnnl }}
     - python {{ python }}
+    - nccl {{ nccl }}         # ['nccl' in str(build_str)]
+    - mpich {{ mpich }}       # ['mpich' in str(build_str)]
 
   run:
     - {{ pin_compatible('glog', max_pin='x.x') }}
     - {{ pin_compatible('numpy', max_pin='x.x') }}
     - {{ pin_compatible('dnnl', max_pin='x.x') }}
     - cudnn {{ cudnn }}       # ['cudnn' in str(build_str)]
     - python {{ python }}
-    - libprotobuf 3.6.1
-    - libopenblas 0.3.3
+    - nccl {{ nccl }}         # ['nccl' in str(build_str)]
+    - mpich {{ mpich }}       # ['mpich' in str(build_str)]
+    - libprotobuf 3.9.2
+    - libopenblas 0.3.9
     - pillow
     - future
     - tqdm
-    - onnx >=1.3.0
+    - onnx 1.6.0
     - deprecated 1.2.7
 
-
 test:
   source_files:
     - test/python/*.py