Skip to content
Permalink
Browse files

MXNet: use MXEnginePushAsync C API to push horovod operations (#985)

* use PushAsyncPtr API

Signed-off-by: Yuxi Hu <darrenyxhu@gmail.com>

* refactor code

Signed-off-by: Yuxi Hu <darrenyxhu@gmail.com>

* sync with API changes

Signed-off-by: Yuxi Hu <darrenyxhu@gmail.com>

* fix opr_name

Signed-off-by: Yuxi Hu <darrenyxhu@gmail.com>

* update MXNet package in CI

Signed-off-by: Yuxi Hu <darrenyxhu@gmail.com>

* address comments

Signed-off-by: Yuxi Hu <darrenyxhu@gmail.com>
  • Loading branch information...
yuxihu authored and alsrgv committed Apr 19, 2019
1 parent 2770fca commit 98d44899a7e98dd449fdafb9ea53c4cec45e7f68
@@ -8,26 +8,26 @@ repository=823773083436.dkr.ecr.us-east-1.amazonaws.com/buildkite

# list of all the tests
tests=( \
test-cpu-openmpi-py2_7-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_4_0-pyspark2_1_2 \
test-cpu-openmpi-py3_5-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_4_0-pyspark2_1_2 \
test-cpu-openmpi-py3_6-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_4_0-pyspark2_1_2 \
test-cpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2 \
test-cpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2 \
test-cpu-openmpi-py3_6-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2 \
test-cpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-cpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0 \
test-cpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2 \
test-gpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2 \
test-gpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-gpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0 \
test-gpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-mixed-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_5_0-pyspark2_1_2 \
test-cpu-openmpi-py3_5-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_5_0-pyspark2_1_2 \
test-cpu-openmpi-py3_6-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_5_0-pyspark2_1_2 \
test-cpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2 \
test-cpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2 \
test-cpu-openmpi-py3_6-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2 \
test-cpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2 \
test-gpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2 \
test-gpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-gpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
test-mixed-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0 \
)

build_test() {
@@ -9,7 +9,7 @@ ARG PYTHON_VERSION=2.7
ARG TENSORFLOW_PACKAGE=tensorflow==1.12.0
ARG KERAS_PACKAGE=keras==2.2.2
ARG PYTORCH_PACKAGE=torch==1.0.0
ARG MXNET_PACKAGE=mxnet==1.4.0.post0
ARG MXNET_PACKAGE=mxnet==1.5.0b20190412
ARG PYSPARK_PACKAGE=pyspark==2.4.0

# Set default shell to /bin/bash
@@ -11,7 +11,7 @@ ARG PYTHON_VERSION=2.7
ARG TENSORFLOW_PACKAGE=tensorflow-gpu==1.12.0
ARG KERAS_PACKAGE=keras==2.2.2
ARG PYTORCH_PACKAGE=torch==1.0.0
ARG MXNET_PACKAGE=mxnet-cu90==1.4.0.post0
ARG MXNET_PACKAGE=mxnet-cu90==1.5.0b20190412
ARG PYSPARK_PACKAGE=pyspark==2.4.0
ARG HOROVOD_BUILD_FLAGS=HOROVOD_GPU_ALLREDUCE=NCCL
ARG HOROVOD_MIXED_INSTALL=0
@@ -5,7 +5,7 @@ services:
context: .
dockerfile: Dockerfile.test.cpu
privileged: true
test-cpu-openmpi-py2_7-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_4_0-pyspark2_1_2:
test-cpu-openmpi-py2_7-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_5_0-pyspark2_1_2:
extends: test-cpu-base
build:
args:
@@ -15,7 +15,7 @@ services:
KERAS_PACKAGE: keras==2.0.0
PYTORCH_PACKAGE: torch==0.4.0
PYSPARK_PACKAGE: pyspark==2.1.2
test-cpu-openmpi-py3_5-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_4_0-pyspark2_1_2:
test-cpu-openmpi-py3_5-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_5_0-pyspark2_1_2:
extends: test-cpu-base
build:
args:
@@ -25,7 +25,7 @@ services:
KERAS_PACKAGE: keras==2.0.0
PYTORCH_PACKAGE: torch==0.4.0
PYSPARK_PACKAGE: pyspark==2.1.2
test-cpu-openmpi-py3_6-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_4_0-pyspark2_1_2:
test-cpu-openmpi-py3_6-tf1_1_0-keras2_0_0-torch0_4_0-mxnet1_5_0-pyspark2_1_2:
extends: test-cpu-base
build:
args:
@@ -35,7 +35,7 @@ services:
KERAS_PACKAGE: keras==2.0.0
PYTORCH_PACKAGE: torch==0.4.0
PYSPARK_PACKAGE: pyspark==2.1.2
test-cpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2:
test-cpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2:
extends: test-cpu-base
build:
args:
@@ -45,7 +45,7 @@ services:
KERAS_PACKAGE: keras==2.1.2
PYTORCH_PACKAGE: torch==0.4.1
PYSPARK_PACKAGE: pyspark==2.3.2
test-cpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2:
test-cpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2:
extends: test-cpu-base
build:
args:
@@ -55,7 +55,7 @@ services:
KERAS_PACKAGE: keras==2.1.2
PYTORCH_PACKAGE: torch==0.4.1
PYSPARK_PACKAGE: pyspark==2.3.2
test-cpu-openmpi-py3_6-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2:
test-cpu-openmpi-py3_6-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2:
extends: test-cpu-base
build:
args:
@@ -65,7 +65,7 @@ services:
KERAS_PACKAGE: keras==2.1.2
PYTORCH_PACKAGE: torch==0.4.1
PYSPARK_PACKAGE: pyspark==2.3.2
test-cpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
test-cpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
@@ -75,7 +75,7 @@ services:
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
test-cpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
@@ -85,7 +85,7 @@ services:
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py3_6-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
test-cpu-openmpi-py3_6-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
@@ -95,7 +95,7 @@ services:
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0:
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
extends: test-cpu-base
build:
args:
@@ -105,7 +105,8 @@ services:
KERAS_PACKAGE: git+https://github.com/keras-team/keras.git
PYTORCH_PACKAGE: torch-nightly
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0:
MXNET_PACKAGE: mxnet --pre
test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
extends: test-cpu-base
build:
args:
@@ -115,7 +116,8 @@ services:
KERAS_PACKAGE: git+https://github.com/keras-team/keras.git
PYTORCH_PACKAGE: torch-nightly
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
MXNET_PACKAGE: mxnet --pre
test-cpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
@@ -136,7 +138,7 @@ services:
environment:
- CUDA_VISIBLE_DEVICES
privileged: true
test-gpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2:
test-gpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2:
extends: test-gpu-base
build:
args:
@@ -147,7 +149,7 @@ services:
KERAS_PACKAGE: keras==2.1.2
PYTORCH_PACKAGE: torch==0.4.1
PYSPARK_PACKAGE: pyspark==2.3.2
test-gpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_0-pyspark2_3_2:
test-gpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_5_0-pyspark2_3_2:
extends: test-gpu-base
build:
args:
@@ -158,7 +160,7 @@ services:
KERAS_PACKAGE: keras==2.1.2
PYTORCH_PACKAGE: torch==0.4.1
PYSPARK_PACKAGE: pyspark==2.3.2
test-gpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
test-gpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
@@ -168,7 +170,7 @@ services:
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
test-gpu-openmpi-py3_5-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
@@ -178,7 +180,7 @@ services:
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0:
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
extends: test-gpu-base
build:
args:
@@ -190,9 +192,9 @@ services:
TENSORFLOW_PACKAGE: tf-nightly-gpu==1.14.1-dev20190408
KERAS_PACKAGE: git+https://github.com/keras-team/keras.git
PYTORCH_PACKAGE: torch-nightly
MXNET_PACKAGE: mxnet-cu100==1.4.0.post0
MXNET_PACKAGE: mxnet-cu100 --pre
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnet1_4_0-pyspark2_4_0:
test-gpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
extends: test-gpu-base
build:
args:
@@ -204,9 +206,9 @@ services:
TENSORFLOW_PACKAGE: tf-nightly-gpu==1.14.1-dev20190408
KERAS_PACKAGE: git+https://github.com/keras-team/keras.git
PYTORCH_PACKAGE: torch-nightly
MXNET_PACKAGE: mxnet-cu100==1.4.0.post0
MXNET_PACKAGE: mxnet-cu100 --pre
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
test-gpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
@@ -216,7 +218,7 @@ services:
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-mixed-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
test-mixed-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
@@ -84,6 +84,12 @@ MXTemporaryBuffer<T>::MXTemporaryBuffer(int device, int dtype)
this->tensor_ = TensorUtil::New(device, dtype);
}

template <class T>
MXTemporaryBuffer<T>::MXTemporaryBuffer(T* tensor)
: MXTensor<T>(nullptr) {
this->tensor_ = tensor;
}

template <class T> MXTemporaryBuffer<T>::~MXTemporaryBuffer() {
TensorUtil::Free(this->tensor_);
}
@@ -51,6 +51,7 @@ template <class T> class MXTensor : public Tensor {
template <class T> class MXTemporaryBuffer : public MXTensor<T> {
public:
MXTemporaryBuffer(int device, int dtype);
MXTemporaryBuffer(T* tensor);
~MXTemporaryBuffer();
virtual T* tensor() const;
};
Oops, something went wrong.

0 comments on commit 98d4489

Please sign in to comment.
You can’t perform that action at this time.