add mxnet

recipes/mxnet/build.sh

@@ -0,0 +1,84 @@
+#!/bin/env bash
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+   export CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY"
+   if [[ "$target_platform" == "osx-arm64" ]]; then
+       cp config/distribution/darwin_cpu.cmake config.cmake
+   else
+       cp config/distribution/darwin_cpu_mkl.cmake config.cmake
+       export MKL_INCLUDE_DIR=${CONDA_PREFIX}/include
+   fi
+elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
+       # if want to add mkl to cuda builds:
+       CMAKE_ARGS_BLAS=-DUSE_BLAS='mkl'
+       export MKL_INCLUDE_DIR=${CONDA_PREFIX}/include
+   if [[ "$cuda_compiler_version" == "None" ]]; then
+       cp config/distribution/linux_cpu_mkl.cmake config.cmake
+   elif [[ $cuda_compiler_version == 11.0 ]]; then
+       cp config/distribution/linux_cu110.cmake config.cmake
+       export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs
+   elif [[ $cuda_compiler_version == 11.2 ]]; then
+       cp config/distribution/linux_cu112.cmake config.cmake  
+       export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs
+       # setting LD_LIBRARY_PATH to potential fix future cuda builds. 
+       # Details from CI runs (limited to 6 hours):
+       # - run fails around 90% through cmake with around 5.5 hours
+       # - complains about not finding `libnvidia-ml.so.1`
+       # - we have `libnvidia-ml.so` in /usr/local/cuda/lib64/stubs
+       # - we could maybe make symlink here too?
+   fi
+fi
+
+rm -rf build; mkdir build && cd build
+
+# consider adding: -DBUILD_SHARED_LIBS=ON \
+# causing ps-lite to fail?
+# need to maybe add ps-lite pkg to cf
+
+cmake \
+       -DCMAKE_PREFIX_PATH=$PREFIX \
+       -DCMAKE_BUILD_TYPE=Release \
+       -DCMAKE_INSTALL_LIBDIR="lib" \
+       -DCMAKE_INSTALL_PREFIX=$PREFIX \
+       -DCMAKE_INCLUDE_PATH=$PREFIX/include \
+       -DUSE_CPP_PACKAGE=ON \
+       -DUSE_OPENMP=ON \
+       -DBLA_STATIC=OFF \
+       ${CMAKE_ARGS_BLAS} \
+       ${CMAKE_ARGS} \
+..
+
+make -j ${CPU_COUNT}
+make install
+
+cd ../python
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    export MXNET_LIBRARY_PATH=${PREFIX}/lib/libmxnet.dylib
+else
+    export MXNET_LIBRARY_PATH=${PREFIX}/lib/libmxnet.so
+fi
+
+export MXNET_INCLUDE_PATH=${PREFIX}/include/mxnet
+
+${PYTHON} -m pip install . -vv
+
+#
+# if [[ "$OSTYPE" == "darwin"* ]]; then
+#     ln ${PREFIX}/lib/libmxnet.dylib $SP_DIR/mxnet/libmxnet.dylib
+#     find ${PREFIX} | grep libmxnet.dylib | grep -v $PREFIX/lib/libmxnet.dylib | xargs rm -f
+#
+# else
+#     ln ${PREFIX}/lib/libmxnet.so $SP_DIR/mxnet/libmxnet.so
+#     find ${PREFIX} | grep libmxnet.so | grep -v $PREFIX/lib/libmxnet.so | xargs rm -f
+#
+# fi
+#
+
+# Copy the [de]activate scripts to $PREFIX/etc/conda/[de]activate.d.
+# This will allow them to be run on environment activation.
+for CHANGE in "activate" "deactivate"
+do
+    mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d"
+    cp "${RECIPE_DIR}/scripts/${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh"
+done

recipes/mxnet/conda_build_config.yaml

@@ -0,0 +1,4 @@
+MACOSX_DEPLOYMENT_TARGET:  # [osx]
+  - '10.13'  # [osx]
+cudnn:
+  - '8'

recipes/mxnet/meta.yaml

@@ -0,0 +1,171 @@
+{% set name = "mxnet" %}
+{% set version = "2.0.0.beta1.rc1" %}
+{% set sha256 = "81ea41d64fd0fd2c984726f0c597b006e7c1f5a9cbbf4d44a1a17af787775345" %}
+{% set build_number = "0" %}
+{% set proc_type = "cuda" if cuda_compiler_version != "None" else "cpu" %}
+
+package:
+  name: mxnet
+  version: {{ version }}
+
+source:
+  - url: https://github.com/apache/incubator-{{ name }}/archive/refs/tags/{{ version }}.tar.gz
+    sha256: {{ sha256 }}
+  # build as provided upstream:
+  # see some discussion here:
+  # https://github.com/apache/incubator-mxnet/issues/19610
+  - url: https://github.com/dmlc/dmlc-core/archive/5df8305.tar.gz
+    sha256: a8046f752f36005564d2924b4b6f73e1aea3cce7ff10f9e19d99ad6a22a045b2
+    folder: 3rdparty/dmlc-core
+  - url: https://github.com/dmlc/ps-lite/archive/34fd45c.tar.gz
+    sha256: ec5d5baab8bbf0c3983ad5f18d7f963f15ae7cd4d154ec204b03c1dceccf209b
+    folder: 3rdparty/ps-lite
+  - url: https://github.com/dmlc/dlpack/archive/3efc489.tar.gz
+    sha256: b59586ce69bcf3efdbf3cf4803fadfeaae4948044e2b8d89cf912194cf28f233
+    folder: 3rdparty/dlpack
+  - url: https://github.com/google/googletest/archive/eb9225c.tar.gz
+    sha256: a4cb4b0c3ebb191b798594aca674ad47eee255dcb4c26885cf7f49777703484f
+    folder: 3rdparty/googletest
+  - url: https://github.com/apache/incubator-tvm/archive/efdac94.tar.gz
+    sha256: 656c38d66baeab885b0212602148bb55ad3cf7d22f52ded8ece53f88e103a2f5
+    folder: 3rdparty/tvm
+  - url: https://github.com/onnx/onnx-tensorrt/archive/2eb74d9.tar.gz
+    sha256: df99819727445c247fb5c21c2fd825ded3269376867457ae84fa6d6f1c0ae331
+    folder: 3rdparty/onnx-tensorrt
+  - url: https://github.com/NVlabs/cub/archive/0158fa1.tar.gz
+    sha256: 43424c4c17a997d1d730c89ec14688671245de7941e02b388d7d3df6ea53777a
+    folder: 3rdparty/nvidia_cub
+  - url: https://github.com/nih-at/libzip/releases/download/v1.8.0/libzip-1.8.0.tar.gz
+    sha256: 30ee55868c0a698d3c600492f2bea4eb62c53849bcf696d21af5eb65f3f3839e
+    folder: 3rdparty/libzip
+  - url: https://github.com/kpu/intgemm/archive/8f28282.tar.gz
+    sha256: bc8bd8015613a13747eb769876385ec53e8c1ea7ae3f8414521dc53b8fcdfc65
+    folder: 3rdparty/intgemm
+  # this onednn folder is unneeded, but somewhere cmake looks for it...
+  # so keep it for now
+  - url: https://github.com/oneapi-src/oneDNN/archive/f40443c.tar.gz
+    sha256: da1dd1bb68ce3318db11e57971f975653e84adcb7d65cd879e558f2245e6de9a
+    folder: 3rdparty/onednn
+  - patches:
+    # must be in order
+    # start with onednn and shared stuff
+    # 0001 will stop it from building onednn
+    - patches/0001-use-existing-onednn-and-make-shared.patch
+    # it still builds other libraries above
+    - patches/0002-edit-setup-py.patch
+    # then aarch64 and ppc64le specific
+    # then osx-64 specific 
+    # then linux-64 specific
+    - patches/0003-put-dlpack-away.patch  # [unix and (target_platform != "osx-arm64")]
+    - patches/0004-put-dmlc-away-maybe.patch  # [unix and x86_64]
+    - patches/0005-put-tvm-away-maybe.patch  # [unix and x86_64]
+
+build:
+  number: {{ build_number }}
+  # to make build faster in staged-recipes
+  skip: True  # [win]
+  skip: True  # [py<39]
+  skip: True  # [cuda_compiler_version in ("10.2", "11.0", "11.1")]
+  string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ proc_type }}  # [cuda_compiler_version == "None"]
+  string: py{{ CONDA_PY }}{{ cuda_compiler_version|replace('.', '') }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ proc_type }}  # [cuda_compiler_version != "None"]
+  run_exports:
+    # ensure matching proc-type
+    - mxnet =*=*_{{ proc_type }}
+
+requirements:
+  build:
+    - {{ compiler('c') }}
+    - {{ compiler('cxx') }}
+    - {{ compiler('fortran') }}
+    - {{ compiler('cuda') }}  # [cuda_compiler_version != "None"]
+    - make
+    - cmake
+    - ccache
+    - pkgconfig
+    - pkg-config
+
+  host:
+    # for library pkg, libmxnet
+    - openblas
+    - llvm-openmp  # [unix]
+    - mkl  # [target_platform != "osx-arm64"]
+    - mkl-include  # [target_platform != "osx-arm64"]
+    # something broke between libopencv and libproto.. something
+    - libopencv  <4.5.5
+    - graphviz
+    - onednn
+    - cudnn  # [cuda_compiler_version != "None"]
+    - nccl  # [cuda_compiler_version != "None"]
+    - cutensor  # [cuda_compiler_version != "None"]
+    - libjpeg-turbo
+    - libprotobuf
+    # for python pkg, mxnet
+    - python
+    - pip
+    - setuptools
+    - requests >=2.20.0,<3
+    # for 3rdparty stuff
+    - dlpack  # [unix and (target_platform != "osx-arm64")]
+    - dmlc  # [unix and (target_platform != "osx-arm64")]
+    - libtvm  # [unix and x86_64]
+    - tvm-py  # [unix and x86_64]
+    # - nnvm  # [unix and x86_64]
+
+  run:
+    # for library pkg, libmxnet 
+    - {{ pin_compatible('openblas') }}
+    - {{ pin_compatible('llvm-openmp') }}  # [unix]
+    - {{ pin_compatible('mkl') }}  # [target_platform != "osx-arm64"]
+    - {{ pin_compatible('libopencv') }}
+    - {{ pin_compatible('graphviz') }}
+    - {{ pin_compatible('onednn') }}
+    - {{ pin_compatible('cudnn') }}  # [cuda_compiler_version != "None"]
+    - {{ pin_compatible('nccl') }}  # [cuda_compiler_version != "None"]
+    - {{ pin_compatible('cutensor') }}  # [cuda_compiler_version != "None"]
+    - {{ pin_compatible('libjpeg-turbo') }}
+    - {{ pin_compatible('libprotobuf') }}
+    # for python pkg, mxnet
+    - python
+    - numpy >=1.17
+    - requests >=2.20.0,<3
+    - python-graphviz >=0.8.1,<0.9.0
+    # for 3rdparty stuff
+    - {{ pin_compatible('dlpack') }}  # [unix and (target_platform != "osx-arm64")]
+    - {{ pin_compatible('dmlc') }}  # [unix and (target_platform != "osx-arm64")]
+    - {{ pin_compatible('libtvm') }}  # [unix and x86_64]
+    - {{ pin_compatible('tvm-py') }}  # [unix and x86_64]
+    # - {{ pin_compatible('nnvm') }}  # [unix and x86_64]
+
+test:
+  imports:
+    - mxnet
+  requires:
+    - pip
+  commands:
+    - pip check
+    - test -f "${PREFIX}/lib/libmxnet${SHLIB_EXT}"  # [unix]
+    - python -c "import mxnet as mx; print(mx.runtime.feature_list())"
+
+about:
+  home: http://mxnet.io
+  license: Apache-2.0
+  license_family: Apache
+  license_file: LICENSE
+  summary: MXNet is a deep learning framework designed for both efficiency and flexibility
+  description: |
+    Apache MXNet (incubating) is a deep learning framework designed for both
+    efficiency and flexibility. It allows you to mix symbolic and imperative
+    programming to maximize efficiency and productivity. At its core, MXNet
+    contains a dynamic dependency scheduler that automatically parallelizes both
+    symbolic and imperative operations on the fly. A graph optimization layer on
+    top of that makes symbolic execution fast and memory efficient. MXNet is
+    portable and lightweight, scaling effectively to multiple GPUs and multiple
+    machines. MXNet is also more than a deep learning project. It is also a
+    collection of blue prints and guidelines for building deep learning systems,
+    and interesting insights of DL systems for hackers.
+  doc_url: https://mxnet.incubator.apache.org/
+  dev_url: https://github.com/apache/incubator-mxnet
+
+extra:
+  recipe-maintainers:
+    - ngam

recipes/mxnet/patches/0001-use-existing-onednn-and-make-shared.patch

@@ -0,0 +1,63 @@
+From bf3f559d0019db4675473d2ed9c88da6aa111683 Mon Sep 17 00:00:00 2001
+From: ngam <67342040+ngam@users.noreply.github.com>
+Date: Sat, 22 Jan 2022 18:55:03 -0500
+Subject: [PATCH 1/5] use existing onednn and make shared
+
+---
+ CMakeLists.txt | 14 +++++---------
+ 1 file changed, 5 insertions(+), 9 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 196e0078a..780392ea1 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -316,7 +316,7 @@ if(USE_ONEDNN)
+     set(ONEDNN_BUILD_EXAMPLES OFF CACHE INTERNAL "" FORCE)
+     set(ONEDNN_ARCH_OPT_FLAGS "" CACHE INTERNAL "" FORCE)
+     set(ONEDNN_ENABLE_JIT_PROFILING OFF CACHE INTERNAL "" FORCE)
+-    set(ONEDNN_LIBRARY_TYPE STATIC CACHE INTERNAL "" FORCE)
++    set(ONEDNN_LIBRARY_TYPE SHARED CACHE INTERNAL "" FORCE)
+     set(ONEDNN_ENABLE_CONCURRENT_EXEC ON CACHE INTERNAL "" FORCE)
+     set(ONEDNN_ENABLE_PRIMITIVE_CACHE ON CACHE INTERNAL "" FORCE)
+
+@@ -324,15 +324,11 @@ if(USE_ONEDNN)
+       set(ONEDNN_CPU_RUNTIME SEQ CACHE INTERNAL "" FORCE)
+     endif()
+
+-    set(CMAKE_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}/onednn")
+-    add_subdirectory(3rdparty/onednn)
++    set(CMAKE_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
+   endfunction()
+   load_onednn()
+-  include_directories(3rdparty/onednn/include)
+-  include_directories(${PROJECT_BINARY_DIR}/3rdparty/onednn/include)
+   add_definitions(-DMXNET_USE_ONEDNN=1)
+   list(APPEND mxnet_LINKER_LIBS dnnl)
+-  set_target_properties(dnnl PROPERTIES CXX_CLANG_TIDY "")  # don't lint 3rdparty dependency
+ endif()
+
+ if(USE_CPP_PACKAGE)
+@@ -699,7 +695,7 @@ if(UNIX)
+   if(MXNET_BUILD_SHARED_LIBS)
+     add_library(mxnet SHARED ${SOURCE})
+   else()
+-    add_library(mxnet STATIC ${SOURCE})
++    add_library(mxnet SHARED ${SOURCE})
+   endif()
+   target_link_libraries(mxnet PUBLIC mshadow)
+   target_link_libraries(mxnet PUBLIC ${CMAKE_DL_LIBS})
+@@ -882,9 +878,9 @@ endif()
+ if(USE_ONEDNN)
+     add_custom_command(TARGET mxnet POST_BUILD
+       COMMAND ${CMAKE_COMMAND} -E copy
+-      ${CMAKE_BINARY_DIR}/3rdparty/onednn/include/oneapi/dnnl/dnnl_config.h  ${CMAKE_SOURCE_DIR}/include/onednn/oneapi/dnnl/
++      ${CMAKE_INSTALL_PREFIX}/include/dnnl_config.h  ${CMAKE_SOURCE_DIR}/include/onednn/oneapi/dnnl/
+       COMMAND ${CMAKE_COMMAND} -E copy
+-      ${CMAKE_BINARY_DIR}/3rdparty/onednn/include/oneapi/dnnl/dnnl_version.h  ${CMAKE_SOURCE_DIR}/include/onednn/oneapi/dnnl/)
++      ${CMAKE_INSTALL_PREFIX}/include/dnnl_version.h  ${CMAKE_SOURCE_DIR}/include/onednn/oneapi/dnnl/)
+ endif()
+
+ if(USE_INTGEMM)
+-- 
+2.34.1
+

recipes/mxnet/patches/0002-edit-setup-py.patch

@@ -0,0 +1,25 @@
+From 67375a7ba098885591eed781c8c38c1e7d0ff356 Mon Sep 17 00:00:00 2001
+From: ngam <67342040+ngam@users.noreply.github.com>
+Date: Sat, 22 Jan 2022 20:36:55 -0500
+Subject: [PATCH 2/5] edit setup py
+
+---
+ python/setup.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/python/setup.py b/python/setup.py
+index ce1130ba8..08769aca9 100644
+--- a/python/setup.py
++++ b/python/setup.py
+@@ -121,7 +121,7 @@ setup(name='mxnet',
+       version=__version__,
+       description=open(os.path.join(CURRENT_DIR, 'README.md')).read(),
+       packages=find_packages(),
+-      data_files=[('mxnet', [LIB_PATH[0]])],
++      data_files=[('mxnet', [])],
+       url='https://github.com/apache/incubator-mxnet',
+       ext_modules=config_cython(),
+       classifiers=[
+-- 
+2.34.1
+

recipes/mxnet/patches/0003-put-dlpack-away.patch

@@ -0,0 +1,33 @@
+From 6fb748ca325387ca38849d15742bee2fbe0cad29 Mon Sep 17 00:00:00 2001
+From: ngam <67342040+ngam@users.noreply.github.com>
+Date: Sat, 22 Jan 2022 20:55:20 -0500
+Subject: [PATCH 3/5] put dlpack away
+
+---
+ CMakeLists.txt | 3 +--
+ 1 file changed, 1 insertion(+), 2 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 780392ea1..4c6da9af4 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -342,7 +342,6 @@ include_directories("include")
+ include_directories("3rdparty/tvm/nnvm/include")
+ include_directories("3rdparty/tvm/include")
+ include_directories("3rdparty/dmlc-core/include")
+-include_directories("3rdparty/dlpack/include")
+
+ if(UNIX)
+   find_library(RTLIB rt)
+@@ -939,7 +938,7 @@ install(TARGETS mxnet
+   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+           COMPONENT   MXNET_Development
+ )
+-install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dlpack/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
++install(DIRECTORY ${CMAKE_INSTALL_PREFIX}/include/dlpack/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/mshadow/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mshadow)
+ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mxnet)
+-- 
+2.34.1
+