Skip to content
Permalink
Browse files

MLSL integration (#908)

Signed-off-by: Yana Shchyokotova <yana.shchyokotova@intel.com>
  • Loading branch information...
shirosankaku authored and alsrgv committed May 30, 2019
1 parent 60506fb commit 7d9d053233f37a8b0a833037c36983a63284bd10
@@ -20,6 +20,7 @@ tests=( \
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-mpich-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_1-pyspark2_4_0 \
test-cpu-mlsl-py3_6-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_1-pyspark2_3_2 \
test-gpu-openmpi-py3_5-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_1-pyspark2_3_2 \
test-gpu-openmpi-py2_7-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_1-pyspark2_4_0 \
@@ -1,7 +1,7 @@
ARG UBUNTU_VERSION=16.04
FROM ubuntu:${UBUNTU_VERSION}

# Arguments for the build. UBUNTU_VERSION needs to be repeated becaus
# Arguments for the build. UBUNTU_VERSION needs to be repeated because
# the first usage only applies to the FROM tag.
ARG UBUNTU_VERSION=16.04
ARG MPI_KIND=OpenMPI
@@ -47,13 +47,50 @@ RUN if [[ ${MPI_KIND} == "OpenMPI" ]]; then \
wget -O /tmp/openmpi-3.0.0-bin.tar.gz https://github.com/uber/horovod/files/1596799/openmpi-3.0.0-bin.tar.gz && \
cd /usr/local && tar -zxf /tmp/openmpi-3.0.0-bin.tar.gz && ldconfig && \
echo "mpirun -allow-run-as-root -np 2 -H localhost:2 -bind-to none -map-by slot -mca mpi_abort_print_stack 1" > /mpirun_command; \
elif [[ ${MPI_KIND} == "MLSL" ]]; then \
wget -O /tmp/l_mlsl_2018.3.008.tgz https://github.com/intel/MLSL/releases/download/IntelMLSL-v2018.3-Preview/l_mlsl_2018.3.008.tgz && \
cd /tmp && tar -zxf /tmp/l_mlsl_2018.3.008.tgz && \
/tmp/l_mlsl_2018.3.008/install.sh -s -d /usr/local/mlsl && \
wget https://raw.githubusercontent.com/intel/MLSL/master/mpirt_2019/include/mpi.h -P /usr/local/mlsl/intel64/include && \
wget https://raw.githubusercontent.com/intel/MLSL/master/mpirt_2019/include/mpio.h -P /usr/local/mlsl/intel64/include && \
wget https://raw.githubusercontent.com/intel/MLSL/master/mpirt_2019/include/mpicxx.h -P /usr/local/mlsl/intel64/include && \
wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/bin/mpicc -P /usr/local/mlsl/intel64/bin && \
chmod +x /usr/local/mlsl/intel64/bin/mpicc && \
wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/bin/mpicxx -P /usr/local/mlsl/intel64/bin && \
chmod +x /usr/local/mlsl/intel64/bin/mpicxx && \
wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/bin/mpigcc -P /usr/local/mlsl/intel64/bin && \
chmod +x /usr/local/mlsl/intel64/bin/mpigcc && \
wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/bin/mpigxx -P /usr/local/mlsl/intel64/bin && \
chmod +x /usr/local/mlsl/intel64/bin/mpigxx && \
wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/lib/libmpicxx.so -P /usr/local/mlsl/intel64/lib && \
chmod +x /usr/local/mlsl/intel64/lib/libmpicxx.so && \
#wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/bin/mpicc -P /usr/local/bin && \
#chmod +x /usr/local/bin/mpicc && \
#wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/bin/mpicxx -P /usr/local/bin && \
#chmod +x /usr/local/bin/mpicxx && \
#wget https://raw.githubusercontent.com/AlekseyMarchuk/MLSL/master/mpirt_2019/bin/mpigcc -P /usr/local/bin && \
#chmod +x /usr/local/bin/mpigcc && \
echo ". /usr/local/mlsl/intel64/bin/mlslvars.sh \"thread\"; \
echo \"mpirun is \$(which mpirun)\"; \
echo \"this file is \$(cat /mpirun_command_script)\"; \
echo \"LD_LIBRARY_PATH is \$(echo \$LD_LIBRARY_PATH)\"; \
echo \"mlsl links with \$(ldd /usr/local/mlsl/intel64/lib/libmlsl.so)\"; \
mpirun -np 2 -ppn 2 -hosts localhost \$@" > /mpirun_command_script && \
chmod +x /mpirun_command_script && \
echo "-L/usr/local/mlsl/intel64/lib/thread -lmpi -I/usr/local/mlsl/intel64/include" > /mpicc_mlsl && \
chmod +x /mpicc_mlsl && \
echo "/mpirun_command_script" > /mpirun_command; \
else \
apt-get install -y mpich && \
echo "mpirun -np 2" > /mpirun_command; \
fi

# Install mpi4py.
RUN pip install mpi4py
RUN if [[ ${MPI_KIND} == "MLSL" ]]; then \
export I_MPI_ROOT=/usr/local/mlsl; \
export MPICC=/usr/local/mlsl/intel64/bin/mpicc; \
fi; \
pip install mpi4py

### END OF CACHE ###
COPY . /horovod
@@ -90,8 +127,41 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 200 && \
update-alternatives --install /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/g++-4.9 200

# Install Horovod.
RUN cd /horovod && python setup.py sdist
RUN pip install -v /horovod/dist/horovod-*.tar.gz
RUN if [[ ${MPI_KIND} == "MLSL" ]]; then \
if [ -z "${LD_LIBRARY_PATH:-}" ]; then \
export LD_LIBRARY_PATH=""; \
fi; \
if [ -z "${PYTHONPATH:-}" ]; then \
export PYTHONPATH=""; \
fi; \
source /usr/local/mlsl/intel64/bin/mlslvars.sh "thread"; \
export I_MPI_ROOT=/usr/local/mlsl; \
export PIP_HOROVOD_MPICXX_SHOW=/usr/local/mlsl/intel64/bin/mpicxx; \
echo "horovod python setup.py sdist, mpicxx is $(which mpicxx)"; \
cd /horovod && python setup.py sdist; \
else \
cd /horovod && python setup.py sdist; \
fi

#RUN if [[ ${MPI_KIND} == "MLSL" ]]; then \
# source /usr/local/mlsl/intel64/bin/mlslvars.sh "thread"; \
# fi; \
# pip install -v /horovod/dist/horovod-*.tar.gz

RUN if [[ ${MPI_KIND} == "MLSL" ]]; then \
if [ -z "${LD_LIBRARY_PATH:-}" ]; then \
export LD_LIBRARY_PATH=""; \
fi; \
if [ -z "${PYTHONPATH:-}" ]; then \
export PYTHONPATH=""; \
fi; \
source /usr/local/mlsl/intel64/bin/mlslvars.sh "thread"; \
echo "pip install horovod, mpicxx is $(which mpicxx)"; \
pip install -v /horovod/dist/horovod-*.tar.gz; \
else \
pip install -v /horovod/dist/horovod-*.tar.gz; \
fi


# Remove GCC pinning
RUN update-alternatives --remove gcc /usr/bin/gcc-4.9 && \
@@ -129,6 +129,16 @@ services:
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-mlsl-py3_6-tf1_12_0-keras2_2_2-torch1_0_0-mxnet1_4_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
MPI_KIND: MLSL
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.12.0
KERAS_PACKAGE: keras==2.2.2
PYTORCH_PACKAGE: torch==1.0.0
PYSPARK_PACKAGE: pyspark==2.4.0

test-gpu-base:
build:
@@ -0,0 +1,55 @@
## Horovod with Intel(R) MLSL

To use Horovod with the Intel(R) Machine Learning Scaling Library (Intel(R) MLSL), follow the steps below.

1. Install [Intel MLSL](https://github.com/intel/MLSL).

To install Intel MLSL, follow [these steps](https://github.com/intel/MLSL/blob/master/README.md).

Source mlslvars.sh to start using Intel MLSL. Two modes are available: `process` (default)
and `thread`. Use the `thread` mode if you are going to set more than zero MLSL servers via MLSL_NUM_SERVERS
environment variable.

```bash
$ source <install_dir>/intel64/bin/mlslvars.sh [mode]
```

2. Install the [Intel(R) MPI Library](https://software.intel.com/en-us/mpi-library).

To install the Intel MPI Library, follow [these steps](https://software.intel.com/en-us/mpi-library/documentation/get-started).

Source the mpivars.sh script to establish the proper environment settings.

```bash
$ source <installdir_MPI>/intel64/bin/mpivars.sh release_mt
```

3. Install Horovod from source code.

```bash
$ python setup.py build
$ python setup.py install
```

**Advanced:** You can specify the affinity for BackgroundThread with the HOROVOD_MLSL_BGT_AFFINITY environment variable.
See the instructions below.

Set Horovod background thread affinity:
```bash
$ export HOROVOD_MLSL_BGT_AFFINITY=c0
```
where c0 is a core ID to attach background thread to.

Set the number of MLSL servers:
```bash
$ export MLSL_NUM_SERVERS=X
```
where X is the number of cores you’d like to dedicate for driving communication. This means that for every rank there are X MLSL
servers available.

Set MLSL servers affinity:
```bash
$ export MLSL_SERVER_AFFINITY=c1,c2,..,cX
```
where c1,c2,..,cX are core IDs dedicated to MLSL servers (uses X ‘last’ cores by default). This variable sets affinity for all
MLSL servers (MLSL_NUM_SERVERS * Number of ranks per node) that are available for all the ranks running on one node.
@@ -1,4 +1,5 @@
// Copyright 2018 Uber Technologies, Inc. All Rights Reserved.
// Modifications copyright (C) 2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -49,6 +50,9 @@ namespace common {
#define NCCL_BCAST "NCCL_BCAST"
#define COPY_ALLGATHER_OUTPUT "COPY_ALLGATHER_OUTPUT"
#define ALLOCATE_SHARED_BUFFER "ALLOCATE_SHARED_BUFFER"
#define MLSL_ALLREDUCE "MLSL_ALLREDUCE"
#define MLSL_ALLGATHER "MLSL_ALLGATHER"
#define MLSL_BCAST "MLSL_BCAST"

// Device ID used for CPU.
#define CPU_DEVICE_ID (-1)
@@ -1,6 +1,7 @@
// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
// Modifications copyright (C) 2019 Uber Technologies, Inc.
// Modifications copyright (C) 2019, NVIDIA CORPORATION. All rights reserved.
// Modifications copyright (C) 2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -55,6 +56,10 @@
#include "ops/ddl_operations.h"
#endif

#if HAVE_MLSL
#include "ops/mlsl_operations.h"
#endif

/*
* Allreduce, Allgather and Broadcast Ops.
*
@@ -103,6 +108,10 @@ NCCLContext nccl_context;
DDLContext ddl_context;
#endif

#if HAVE_MLSL
MLSLContext mlsl_context;
#endif

std::unique_ptr<OperationManager> op_manager;

// For clarify in argument lists.
@@ -149,10 +158,16 @@ OperationManager* CreateOperationManager(HorovodGlobalState& state) {
#endif
#endif

#if HAVE_MLSL
allreduce_ops.push_back(std::shared_ptr<AllreduceOp>(new MLSLAllreduce(&mlsl_context, &state)));
allgather_ops.push_back(std::shared_ptr<AllgatherOp>(new MLSLAllgather(&mlsl_context, &mpi_context, &state)));
broadcast_ops.push_back(std::shared_ptr<BroadcastOp>(new MLSLBroadcast(&mlsl_context, &state)));
#else
// Default operations, always enabled but last to be checked.
allreduce_ops.push_back(std::shared_ptr<AllreduceOp>(new MPIAllreduce(&mpi_context, &state)));
allgather_ops.push_back(std::shared_ptr<AllgatherOp>(new MPIAllgather(&mpi_context, &state)));
broadcast_ops.push_back(std::shared_ptr<BroadcastOp>(new MPIBroadcast(&mpi_context, &state)));
#endif
std::shared_ptr<ErrorOp> error_op(new ErrorOp(&state));

return new OperationManager(&state.param_manager, allreduce_ops, allgather_ops, broadcast_ops, error_op);
@@ -854,6 +869,7 @@ void CoordinateCacheAndState(CacheCoordinator& cache_coordinator,
// otherwise we may end up dispatching many blocked threads and never make
// progress if we have a thread pool limit.
bool RunLoopOnce(HorovodGlobalState& state, MPIContext& ctx, bool is_coordinator);

void BackgroundThreadLoop(HorovodGlobalState& state, MPIContext& ctx) {
// Initialize MPI if it was not initialized. This must happen on the
// background thread, since not all MPI implementations support being called
@@ -921,6 +937,9 @@ void BackgroundThreadLoop(HorovodGlobalState& state, MPIContext& ctx) {
// Get MPI size to determine how many tensors to wait for before reducing.
int size;
MPI_Comm_size(ctx.mpi_comm, &size);
#if HAVE_MLSL
mlsl_context.Init(size);
#endif
if (is_coordinator) {
LOG(INFO) << "Started Horovod with " << size << " processes";
}
@@ -1101,6 +1120,9 @@ void BackgroundThreadLoop(HorovodGlobalState& state, MPIContext& ctx) {
while (RunLoopOnce(state, ctx, is_coordinator))
;

#if HAVE_MLSL
mlsl_context.Finalize();
#endif
LOG(DEBUG, rank) << "Shutting down background thread";

// Signal that shutdown has been requested.
@@ -1587,6 +1609,8 @@ void InitializeHorovodOnce(const int* ranks, int nranks) {
while (!horovod_global.initialization_done) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}

LOG(DEBUG) << "Background thread init done";
}

} // namespace
@@ -43,6 +43,7 @@ namespace common {
#define HOROVOD_HIERARCHICAL_ALLREDUCE "HOROVOD_HIERARCHICAL_ALLREDUCE"
#define HOROVOD_HIERARCHICAL_ALLGATHER "HOROVOD_HIERARCHICAL_ALLGATHER"
#define HOROVOD_CACHE_CAPACITY "HOROVOD_CACHE_CAPACITY"
#define HOROVOD_MLSL_BGT_AFFINITY "HOROVOD_MLSL_BGT_AFFINITY"

// Check that Horovod is initialized.
Status CheckInitialized();

0 comments on commit 7d9d053

Please sign in to comment.
You can’t perform that action at this time.