Enable MKL-DNN by default in pip packages (#16899)

* enable mkldnn by default in pip packages

* add make file for native build

* rm *mkl build scripts

* remove *mkl variants

* Change readme/cd

* clear native build configurations

* fix static build test in ci

* git mv linux_cu102mkl.mk linux_cu102.mk

* fix merge conflict

cd/Jenkinsfile_cd_pipeline

@@ -36,7 +36,7 @@ pipeline {
 
   parameters {
     // Release parameters
-    string(defaultValue: "cpu,mkl,cu90,cu90mkl,cu92,cu92mkl,cu100,cu100mkl,cu101,cu101mkl,cu102,cu102mkl", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
+    string(defaultValue: "cpu,native,cu90,cu92,cu100,cu101,cu102", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
     booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: "RELEASE_BUILD")
   }
 

cd/Jenkinsfile_release_job

@@ -43,7 +43,7 @@ pipeline {
     // any disruption caused by different COMMIT_ID values chaning the job parameter configuration on
     // Jenkins.
     string(defaultValue: "mxnet_lib/static", description: "Pipeline to build", name: "RELEASE_JOB_TYPE")
-    string(defaultValue: "cpu,mkl,cu90,cu90mkl,cu92,cu92mkl,cu100,cu100mkl,cu101,cu101mkl,cu102,cu102mkl", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
+    string(defaultValue: "cpu,native,cu90,cu92,cu100,cu101,cu102", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
     booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: "RELEASE_BUILD")
   }
 

cd/README.md

@@ -19,28 +19,23 @@
 
 ## Introduction
 
-MXNet aims to support a variety of frontends, e.g. Python, Java, Perl, R, etc. as well as environments (Windows, Linux, Mac, with or without GPU, with or without MKL support, etc.). This package contains a small continuous delivery (CD) framework used to automate the delivery nightly and release builds across our delivery channels.
+MXNet aims to support a variety of frontends, e.g. Python, Java, Perl, R, etc. as well as environments (Windows, Linux, Mac, with or without GPU, with or without MKL-DNN support, etc.). This package contains a small continuous delivery (CD) framework used to automate the delivery nightly and release builds across our delivery channels.
 
 <!-- TODO: Add links to the actual jobs, once this is live on PROD -->
 
 The CD process is driven by the [CD pipeline job](Jenkinsfile_cd_pipeline), which orchestrates the order in which the artifacts are delivered. For instance, first publish the libmxnet library before publishing the pip package. It does this by triggering the [release job](Jenkinsfile_release_job) with a specific set of parameters for each delivery channel. The release job executes the specific release pipeline for a delivery channel across all MXNet *variants*.
 
-A variant is a specific environment or features for which MXNet is compiled. For instance CPU, GPU with CUDA v10.0, CUDA v9.0 with MKL support, etc. 
+A variant is a specific environment or features for which MXNet is compiled. For instance CPU, GPU with CUDA v10.0, CUDA v9.0 with MKL-DNN support, etc. 
 
-Currently, 10 variants are supported:
+Currently, below variants are supported. All of these variants except native have MKL-DNN backend enabled.
 
 * *cpu*: CPU
-* *mkl*: CPU w/ MKL
+* *native*: CPU without MKL-DNN
 * *cu90*: CUDA 9.0
-* *cu90mkl*: CUDA 9.0 w/ MKL-DNN
 * *cu92*: CUDA 9.2
-* *cu92mkl*: CUDA 9.2 w/ MKL-DNN
 * *cu100*: CUDA 10
-* *cu100mkl*: CUDA 10 w/ MKL-DNN
 * *cu101*: CUDA 10.1
-* *cu101mkl*: CUDA 10.1 w/ MKL-DNN
 * *cu102*: CUDA 10.2
-* *cu102mkl*: CUDA 10.2 w/ MKL-DNN
 
 *For more on variants, see [here](https://github.com/apache/incubator-mxnet/issues/8671)*
 

cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy

@@ -31,7 +31,7 @@ licenses = 'licenses/*'
 
 // libmxnet dependencies
 mx_deps = ''
-mx_mkldnn_deps = ''
+mx_native_deps = ''
 
 // library type
 // either static or dynamic - depending on how it links to its dependencies

cd/mxnet_lib/mxnet_lib_pipeline.groovy

@@ -68,7 +68,7 @@ def get_pipeline(mxnet_variant, build_fn) {
 // Returns a string of comma separated resources to be stashed b/w stages
 // E.g. the libmxnet library and any other dependencies
 def get_stash(mxnet_variant) {
-  def deps = mxnet_variant.endsWith('mkl') ? mx_mkldnn_deps : mx_deps
+  def deps = mxnet_variant.endsWith('native') ? mx_native_deps : mx_deps
   return "${libmxnet}, ${licenses}, ${deps}"
 }
 
@@ -120,7 +120,7 @@ def test_gpu_quantization_py3(mxnet_variant) {
 def push(mxnet_variant) {
   node(NODE_LINUX_CPU) {
     ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") {
-      def deps = (mxnet_variant.endsWith('mkl')? mx_mkldnn_deps : mx_deps).replaceAll(',', '')
+      def deps = (mxnet_variant.endsWith('native')? mx_native_deps : mx_deps).replaceAll(',', '')
       ci_utils.unpack_and_init("mxnet_${mxnet_variant}", get_stash(mxnet_variant), false)
       cd_utils.push_artifact(libmxnet, mxnet_variant, libtype, licenses, deps)
     }

cd/mxnet_lib/static/Jenkins_pipeline.groovy

@@ -32,8 +32,8 @@ libmxnet = 'lib/libmxnet.so'
 licenses = 'licenses/*'
 
 // libmxnet dependencies
-mx_deps = 'lib/libgfortran.so.3, lib/libquadmath.so.0'
-mx_mkldnn_deps = 'lib/libgfortran.so.3, lib/libquadmath.so.0, 3rdparty/mkldnn/build/install/include/dnnl_version.h, 3rdparty/mkldnn/build/install/include/dnnl_config.h'
+mx_native_deps = 'lib/libgfortran.so.3, lib/libquadmath.so.0'
+mx_deps = 'lib/libgfortran.so.3, lib/libquadmath.so.0, 3rdparty/mkldnn/build/install/include/dnnl_version.h, 3rdparty/mkldnn/build/install/include/dnnl_config.h'
 
 // library type
 // either static or dynamic - depending on how it links to its dependencies

cd/python/docker/test_python_image.sh

@@ -38,7 +38,7 @@ if [[ $mxnet_variant == cu* ]]; then
     test_conv_params="--gpu"
 fi
 
-if [[ $mxnet_variant == *mkl ]]; then
+if [[ $mxnet_variant == cpu ]]; then
     ${python_cmd} tests/python/mkl/test_mkldnn.py
 fi
 

cd/python/pypi/pypi_package.sh

@@ -18,7 +18,7 @@
 
 set -ex
 
-# variant = cpu, mkl, cu80, cu80mkl, cu100, etc.
+# variant = cpu, native, cu80, cu100, etc.
 export mxnet_variant=${1:?"Please specify the mxnet variant"}
 
 # Due to this PR: https://github.com/apache/incubator-mxnet/pull/14899

cd/utils/artifact_repository.md

@@ -53,13 +53,13 @@ If not set, derived through the value of sys.platform (https://docs.python.org/3
 
 **Variant**
 
-Manually configured through the --variant argument. The current variants are: cpu, mkl, cu80, cu80mk, cu90, cu90mkl, cu92, cu92mkl, cu100, cu100mkl.
+Manually configured through the --variant argument. The current variants are: cpu, native, cu80, cu90, cu92, cu100, and cu101.
 
-As long as the tool is being run from the MXNet code base, the runtime feature detection tool (https://github.com/larroy/mxnet/blob/dd432b7f241c9da2c96bcb877c2dc84e6a1f74d4/docs/api/python/libinfo/libinfo.md) can be used to detect whether the library has been compiled with MKL (library has MKLDNN feature enabled) and/or CUDA support (compiled with CUDA feature enabled).
+As long as the tool is being run from the MXNet code base, the runtime feature detection tool (https://github.com/larroy/mxnet/blob/dd432b7f241c9da2c96bcb877c2dc84e6a1f74d4/docs/api/python/libinfo/libinfo.md) can be used to detect whether the library has been compiled with MKL (library has MKL-DNN feature enabled) and/or CUDA support (compiled with CUDA feature enabled).
 
 If it has been compiled with CUDA support, the output of /usr/local/cuda/bin/nvcc --version can be mined for the exact CUDA version (eg. 8.0, 9.0, etc.).
 
-By knowing which features are enabled on the binary, and if necessary, which CUDA version is installed on the machine, the value for the variant argument can be calculated. Eg. if MKL and CUDA features are enabled, and nvcc reports cuda version 10, then the variant would be cu100mkl. If neither MKL nor CUDA features are enabled, the variant would be cpu. 
+By knowing which features are enabled on the binary, and if necessary, which CUDA version is installed on the machine, the value for the variant argument can be calculated. Eg. if CUDA features are enabled, and nvcc reports cuda version 10, then the variant would be cu100. If neither MKL-DNN nor CUDA features are enabled, the variant would be native. 
 
 **Dependency Linking**
 
@@ -83,7 +83,7 @@ The user must specify the directory to which the artifact should be downloaded.
 
 Example:
 
-`./artifact_repository.py --pull --static --variant=cu90mkl ./dist`
+`./artifact_repository.py --pull --static --variant=cu90 ./dist`
 
 This would result in the following directory structure:
 

cd/utils/artifact_repository.py

@@ -291,9 +291,9 @@ def probe_cpu_variant(mxnet_features: Dict[str, bool]) -> str:
     :return: Either cpu, or mkl as the variant
     """
     logger.debug('Determining cpu variant')
-    if mxnet_features['MKLDNN']:
-        logger.debug('variant is: mkl')
-        return 'mkl'
+    if not mxnet_features['MKLDNN']:
+        logger.debug('variant is: native')
+        return 'native'
 
     logger.debug('variant is: cpu')
     return 'cpu'
@@ -312,8 +312,8 @@ def probe_gpu_variant(mxnet_features: Dict[str, bool]) -> Optional[str]:
     cuda_version = get_cuda_version()
     if cuda_version:
         variant = 'cu{}'.format(cuda_version)
-        if mxnet_features['MKLDNN']:
-            variant = '{}mkl'.format(variant)
+        if not mxnet_features['MKLDNN']:
+            RuntimeError('Error determining mxnet variant: MKL-DNN should be enabled for cuda variants')
         logger.debug('variant is: {}'.format(variant))
         return variant
 
@@ -323,7 +323,7 @@ def probe_gpu_variant(mxnet_features: Dict[str, bool]) -> Optional[str]:
 def probe_mxnet_variant(limxnet_path: str) -> Optional[str]:
     """
     Probes the libmxnet library and environment to determine
-    the mxnet variant, eg. cpu, mkl, cu90, cu90mkl, etc.
+    the mxnet variant, eg. cpu, cu90, etc.
     :return:
     """
     logger.debug('Probing for mxnet variant')

cd/utils/docker_tag.sh

@@ -28,15 +28,11 @@ version=${VERSION:-nightly}
 
 if [[ ${mxnet_variant} == "cpu" ]]; then
     tag_suffix="cpu"
-elif [[ ${mxnet_variant} == "mkl" ]]; then
-    tag_suffix="cpu_mkl"
+elif [[ ${mxnet_variant} == "native" ]]; then
+    tag_suffix="native"
 elif [[ ${mxnet_variant} == cu* ]]; then
     tag_suffix="gpu_${mxnet_variant}"
 
-    # *mkl => *_mkl
-    if [[ $tag_suffix == *mkl ]]; then
-        tag_suffix="${tag_suffix:0:${#tag_suffix}-3}_mkl"
-    fi
 else
     echo "Error: Unrecognized mxnet variant: '${mxnet_variant}'."
     exit 1
@@ -49,11 +45,9 @@ if [[ ${is_release} == "true" ]]; then
     if [[ ${mxnet_variant} == "cpu" ]]; then
         echo "latest"
         echo "latest_cpu"
-    elif [[ ${mxnet_variant} == "mkl" ]]; then
-        echo "latest_cpu_mkl"
+    elif [[ ${mxnet_variant} == "native" ]]; then
+        echo "latest_cpu_native"
     elif [[ ${mxnet_variant} == "cu90" ]]; then
         echo "latest_gpu"
-    elif [[ ${mxnet_variant} == "cu90mkl" ]]; then
-        echo "latest_gpu_mkl"
     fi
 fi

cd/utils/mxnet_base_image.sh

@@ -42,7 +42,7 @@ case ${mxnet_variant} in
     cpu)
     echo "ubuntu:16.04"
     ;;
-    mkl)
+    native)
     echo "ubuntu:16.04"
     ;;
     *)

cd/utils/test_artifact_repository.py

@@ -159,40 +159,30 @@ def test_get_cuda_version_not_found(self, mock):
         self.assertIsNone(get_cuda_version())
 
     @patch('artifact_repository.get_libmxnet_features')
-    def test_probe_variant_cpu(self, mock_features):
+    def test_probe_variant_native(self, mock_features):
         """
-        Tests 'cpu' is returned if MKLDNN and CUDA features are OFF
+        Tests 'native' is returned if MKLDNN and CUDA features are OFF
         """
         mock_features.return_value = {'MKLDNN': False, 'CUDA': False}
-        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cpu')
+        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'native')
 
     @patch('artifact_repository.get_libmxnet_features')
-    def test_probe_variant_mkl(self, mock_features):
+    def test_probe_variant_cpu(self, mock_features):
         """
-        Tests 'mkl' is returned if MKLDNN is ON and CUDA is OFF
+        Tests 'cpu' is returned if MKLDNN is ON and CUDA is OFF
         """
         mock_features.return_value = {'MKLDNN': True, 'CUDA': False}
-        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'mkl')
+        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cpu')
 
     @patch('artifact_repository.get_libmxnet_features')
     @patch('artifact_repository.get_cuda_version')
     def test_probe_variant_cuda(self, mock_cuda_version, mock_features):
         """
         Tests 'cu100' is returned if MKLDNN is OFF and CUDA is ON and CUDA version is 10.0
         """
-        mock_features.return_value = {'MKLDNN': False, 'CUDA': True}
-        mock_cuda_version.return_value = '100'
-        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cu100')
-
-    @patch('artifact_repository.get_libmxnet_features')
-    @patch('artifact_repository.get_cuda_version')
-    def test_probe_variant_cuda_mkl(self, mock_cuda_version, mock_features):
-        """
-        Tests 'cu100mkl' is returned if MKLDNN is ON and CUDA is ON and CUDA version is 10.0
-        """
         mock_features.return_value = {'MKLDNN': True, 'CUDA': True}
         mock_cuda_version.return_value = '100'
-        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cu100mkl')
+        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cu100')
 
     @patch('artifact_repository.get_libmxnet_features')
     def test_probe_variant_cuda_returns_none_on_no_features(self, mock_features):

ci/docker/runtime_functions.sh

@@ -1994,28 +1994,28 @@ cd_s3_publish() {
     aws s3 cp ${filepath} s3://apache-mxnet/dist/python/${variant}/${filename} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=43f628fab72838a4f0b929d7f1993b14411f4b0294b011261bc6bd3e950a6822
 }
 
-build_static_scala_mkl() {
+build_static_scala_cpu() {
     set -ex
     pushd .
     scala_prepare
     export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
-    export mxnet_variant=mkl
+    export mxnet_variant=cpu
     ./ci/publish/scala/build.sh
     popd
 }
 
-build_static_python_mkl() {
+build_static_python_cpu() {
     set -ex
     pushd .
-    export mxnet_variant=mkl
+    export mxnet_variant=cpu
     ./ci/publish/python/build.sh
     popd
 }
 
-build_static_python_cu101mkl() {
+build_static_python_cu101() {
     set -ex
     pushd .
-    export mxnet_variant=cu101mkl
+    export mxnet_variant=cu101
     ./ci/publish/python/build.sh
     popd
 }

ci/jenkins/Jenkins_steps.groovy

@@ -656,7 +656,7 @@ def test_static_scala_cpu() {
         ws('workspace/ut-publish-scala-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
-            utils.docker_run("publish.ubuntu1404_cpu", 'build_static_scala_mkl', false)
+            utils.docker_run("publish.ubuntu1404_cpu", 'build_static_scala_cpu', false)
           }
         }
     }
@@ -669,7 +669,7 @@ def test_static_python_cpu() {
         ws('workspace/ut-publish-python-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
-            utils.docker_run("publish.ubuntu1404_cpu", 'build_static_python_mkl', false)
+            utils.docker_run("publish.ubuntu1404_cpu", 'build_static_python_cpu', false)
           }
         }
     }
@@ -695,7 +695,7 @@ def test_static_python_gpu() {
         ws('workspace/ut-publish-python-gpu') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
-            utils.docker_run("publish.ubuntu1404_gpu", 'build_static_python_cu101mkl', true)
+            utils.docker_run("publish.ubuntu1404_gpu", 'build_static_python_cu101', true)
           }
         }
     }

make/staticbuild/darwin_cpu.mk

@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+#
 #-------------------------------------------------------------------------------
 #  Template configuration for compiling mxnet for making python wheel
 #-------------------------------------------------------------------------------
@@ -83,24 +83,8 @@ USE_OPENMP = 0
 USE_OPERATOR_TUNING = 1
 USE_LIBJPEG_TURBO = 1
 
-
-# MKL ML Library for Intel CPU/Xeon Phi
-# Please refer to MKL_README.md for details
-
 # whether use MKL-DNN library
-USE_MKLDNN = 0
-
-# MKL ML Library folder, need to be root for /usr/local
-# Change to User Home directory for standard user
-# For USE_BLAS!=mkl only
-MKLML_ROOT=/usr/local
-
-# whether use MKL2017 library
-USE_MKL2017 = 0
-
-# whether use MKL2017 experimental feature for high performance
-# Prerequisite USE_MKL2017=1
-USE_MKL2017_EXPERIMENTAL = 0
+USE_MKLDNN = 1
 
 # whether use NNPACK library
 USE_NNPACK = 0

make/staticbuild/linux_cpu.mk

@@ -37,7 +37,7 @@ DEBUG = 0
 USE_SIGNAL_HANDLER = 1
 
 # the additional link flags you want to add
-ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
+ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl  -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
 
 # the additional compile flags you want to add
 ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections
@@ -84,23 +84,7 @@ USE_OPERATOR_TUNING = 1
 USE_LIBJPEG_TURBO = 1
 
 # whether use MKL-DNN library
-USE_MKLDNN = 0
-
-
-# MKL ML Library for Intel CPU/Xeon Phi
-# Please refer to MKL_README.md for details
-
-# MKL ML Library folder, need to be root for /usr/local
-# Change to User Home directory for standard user
-# For USE_BLAS!=mkl only
-MKLML_ROOT=/usr/local
-
-# whether use MKL2017 library
-USE_MKL2017 = 0
-
-# whether use MKL2017 experimental feature for high performance
-# Prerequisite USE_MKL2017=1
-USE_MKL2017_EXPERIMENTAL = 0
+USE_MKLDNN = 1
 
 # whether use NNPACK library
 USE_NNPACK = 0