Merge pull request #113 from sony/feature/20181114-fix-multi-gpu-docker

Add openmpi-bin to Multi-GPU docker.
sony · Nov 16, 2018 · 2a1eab7 · 2a1eab7
2 parents 6271031 + 85db96b
commit 2a1eab7
Show file tree

Hide file tree

Showing 19 changed files with 261 additions and 60 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -33,6 +33,12 @@ if(NOT IS_ABSOLUTE ${NNABLA_DIR})
   get_filename_component(NNABLA_DIR ${CMAKE_CURRENT_BINARY_DIR}/${NNABLA_DIR} ABSOLUTE)
 endif()
 
+file(STRINGS ${NNABLA_DIR}/VERSION.txt versions NEWLINE_CONSUME)
+string(REGEX REPLACE [[^([A-z0-9.\-_]+).*$]] [[\1]] version ${versions})
+string(REGEX REPLACE [[^([0-9]+)\.([0-9]+)\.(.*)$]] [[\1]] version_major ${version})
+string(REGEX REPLACE [[^([0-9]+)\.([0-9]+)\.(.*)$]] [[\2]] version_minor ${version})
+string(REGEX REPLACE [[^([0-9]+)\.([0-9]+)\.(.*)$]] [[\3]] version_patch ${version})
+
 if(NOT EXISTS ${CPPLIB_LIBRARY})
   message(FATAL_ERROR "Invalid CPPLIB_LIBRARY :[${CPPLIB_LIBRARY}]")
 endif()
@@ -82,7 +88,7 @@ include(${PROJECT_SOURCE_DIR}/build-tools/cmake/get_cuda_version.cmake)
 
 execute_process(
   COMMAND ${PYTHON_COMMAND}
-          ${CMAKE_CURRENT_SOURCE_DIR}/build-tools/code_generator/generate.py ${CUDA_VERSION} ${CUDNN_VERSION}
+          ${CMAKE_CURRENT_SOURCE_DIR}/build-tools/code_generator/generate.py ${version} ${CUDA_VERSION} ${CUDNN_VERSION}
           OUTPUT_VARIABLE TEST
           RESULT_VARIABLE GENERATE_RESULT)
   message("${TEST}")

diff --git a/build-tools/code_generator/generate.py b/build-tools/code_generator/generate.py
@@ -72,22 +72,24 @@ def generate(argv):
     utils.generate_solver_types(
         solver_info, solver_types, ext_info={}, template=solver_template, output_format='%s.cu')
 
-    if len(argv) == 3:
-        cuda_version = argv[1]
-        cudnn_version = argv[2]
+    if len(argv) == 4:
+        version = argv[1]
+        cuda_version = argv[2]
+        cudnn_version = argv[3]
     else:
+        version = 'unknown'
         cuda_version = 'unknown'
         cudnn_version = 'unknown'
 
     utils.generate_version(template=join(
         base, 'python/src/nnabla_ext/cuda/_version.py.tmpl'),
-        rootdir=base, cuda_version=cuda_version, cudnn_version=cudnn_version)
+        rootdir=base, version=version, cuda_version=cuda_version, cudnn_version=cudnn_version)
     utils.generate_version(template=join(
         base, 'python/src/nnabla_ext/cudnn/_version.py.tmpl'),
-        rootdir=base, cuda_version=cuda_version, cudnn_version=cudnn_version)
+        rootdir=base, version=version, cuda_version=cuda_version, cudnn_version=cudnn_version)
     utils.generate_version(template=join(
         base, 'src/nbla/cuda/version.cpp.tmpl'),
-        rootdir=base, cuda_version=cuda_version, cudnn_version=cudnn_version)
+        rootdir=base, version=version, cuda_version=cuda_version, cudnn_version=cudnn_version)
 
     # Generate function skeletons
     func_src_template = join(

diff --git a/build-tools/make/build-with-docker.mk b/build-tools/make/build-with-docker.mk
@@ -128,7 +128,7 @@ docker_image_nnabla_ext_cuda:
 	BASE=nvidia/cuda:$(CUDA_VERSION_MAJOR).$(CUDA_VERSION_MINOR)-cudnn$(CUDNN_VERSION)-runtime-ubuntu16.04 \
 	&& docker pull $${BASE} \
 	&& cd $(NNABLA_EXT_CUDA_DIRECTORY) \
-	&& cp docker/Dockerfile.runtime Dockerfile \
+	&& cp docker/runtime/Dockerfile.runtime Dockerfile \
 	&& cp $(BUILD_DIRECTORY_WHEEL)/dist/*.whl . \
 	&& echo ADD $(shell basename $(BUILD_DIRECTORY_WHEEL)/dist/*.whl) /tmp/ >>Dockerfile \
 	&& echo RUN pip install /tmp/$(shell basename $(BUILD_DIRECTORY_WHEEL)/dist/*.whl) >>Dockerfile \
@@ -146,7 +146,7 @@ docker_image_nnabla_ext_cuda_multi_gpu: bwd-nnabla-ext-cuda-wheel-multi-gpu
 	BASE=nvidia/cuda:$(CUDA_VERSION_MAJOR).$(CUDA_VERSION_MINOR)-cudnn$(CUDNN_VERSION)-runtime-ubuntu16.04 \
 	&& docker pull $${BASE} \
 	&& cd $(NNABLA_EXT_CUDA_DIRECTORY) \
-	&& cp docker/development/Dockerfile.runtime-multi-gpu Dockerfile \
+	&& cp docker/runtime/Dockerfile.runtime-multi-gpu Dockerfile \
 	&& cp $(BUILD_DIRECTORY_WHEEL)/dist/*.whl . \
 	&& echo ADD $(shell basename $(BUILD_DIRECTORY_WHEEL)/dist/*.whl) /tmp/ >>Dockerfile \
 	&& echo RUN pip install /tmp/$(shell basename $(BUILD_DIRECTORY_WHEEL)/dist/*.whl) >>Dockerfile \
@@ -157,3 +157,79 @@ docker_image_nnabla_ext_cuda_multi_gpu: bwd-nnabla-ext-cuda-wheel-multi-gpu
 	&& rm -f $(shell basename $(BUILD_DIRECTORY_WHEEL)/dist/*.whl) \
 	&& rm -f $(shell basename $(BUILD_EXT_CUDA_DIRECTORY_WHEEL_MULTI_GPU)/dist/*.whl) \
 	&& rm -f Dockerfile
+
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda90
+docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda90:
+	docker pull nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py35-cuda90 \
+		-f nnabla-ext-cuda/docker/py35/cuda90-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda92
+docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda92:
+	docker pull nvidia/cuda:9.2-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py35-cuda92 \
+		-f nnabla-ext-cuda/docker/py35/cuda92-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda100
+docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda100:
+	docker pull nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py35-cuda100 \
+		-f nnabla-ext-cuda/docker/py35/cuda100-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda90
+docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda90:
+	docker pull nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py36-cuda90 \
+		-f nnabla-ext-cuda/docker/py36/cuda90-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda92
+docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda92:
+	docker pull nvidia/cuda:9.2-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py36-cuda92 \
+		-f nnabla-ext-cuda/docker/py36/cuda92-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda100
+docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda100:
+	docker pull nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py36-cuda100 \
+		-f nnabla-ext-cuda/docker/py36/cuda100-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda90
+docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda90:
+	docker pull nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py37-cuda90 \
+		-f nnabla-ext-cuda/docker/py37/cuda90-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda92
+docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda92:
+	docker pull nvidia/cuda:9.2-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py37-cuda92 \
+		-f nnabla-ext-cuda/docker/py37/cuda92-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda100
+docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda100:
+	docker pull nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04
+	docker build --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} \
+		-t $(DOCKER_IMAGE_NAME_BASE)-multi-gpu-py37-cuda100 \
+		-f nnabla-ext-cuda/docker/py37/cuda100-multi-gpu/Dockerfile .
+
+.PHONY: docker_image_nnabla_ext_cuda_multi_gpu_all
+docker_image_nnabla_ext_cuda_multi_gpu_all: \
+	docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda90 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda92 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py35_cuda100 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda90 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda92 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py36_cuda100 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda90 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda92 \
+	docker_image_nnabla_ext_cuda_multi_gpu_py37_cuda100
diff --git a/docker/development/Dockerfile.runtime-multi-gpu b/docker/development/Dockerfile.runtime-multi-gpu
diff --git a/docker/py35/cuda100-multi-gpu/Dockerfile b/docker/py35/cuda100-multi-gpu/Dockerfile
@@ -1,7 +1,7 @@
 FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
     && rm -rf /var/lib/apt/lists/*
 
 RUN umask 0 \

diff --git a/docker/py35/cuda90-multi-gpu/Dockerfile b/docker/py35/cuda90-multi-gpu/Dockerfile
@@ -1,7 +1,7 @@
 FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
     && rm -rf /var/lib/apt/lists/*
 
 RUN umask 0 \

diff --git a/docker/py35/cuda92-multi-gpu/Dockerfile b/docker/py35/cuda92-multi-gpu/Dockerfile
@@ -1,7 +1,7 @@
 FROM nvidia/cuda:9.2-cudnn7-runtime-ubuntu16.04
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
     && rm -rf /var/lib/apt/lists/*
 
 RUN umask 0 \

diff --git a/docker/py36/cuda100-multi-gpu/Dockerfile b/docker/py36/cuda100-multi-gpu/Dockerfile
@@ -1,7 +1,7 @@
 FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
     && rm -rf /var/lib/apt/lists/*
 
 RUN umask 0 \

diff --git a/docker/py36/cuda90-multi-gpu/Dockerfile b/docker/py36/cuda90-multi-gpu/Dockerfile
@@ -1,7 +1,7 @@
 FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
     && rm -rf /var/lib/apt/lists/*
 
 RUN umask 0 \

diff --git a/docker/py36/cuda92-multi-gpu/Dockerfile b/docker/py36/cuda92-multi-gpu/Dockerfile
@@ -1,7 +1,7 @@
 FROM nvidia/cuda:9.2-cudnn7-runtime-ubuntu16.04
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
     && rm -rf /var/lib/apt/lists/*
 
 RUN umask 0 \

diff --git a/docker/py37/cuda100-multi-gpu/Dockerfile b/docker/py37/cuda100-multi-gpu/Dockerfile
@@ -0,0 +1,23 @@
+FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN umask 0 \
+    && mkdir -p /tmp/deps \
+    && cd /tmp/deps \
+    && curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+    && rm -rf Miniconda3-latest-Linux-x86_64.sh \
+    && PATH=/opt/miniconda3/bin:$PATH \
+    && conda install python=3.7 \
+    && conda update -y --all \
+    && conda install pip wheel opencv \
+    && conda clean -y --all \
+    && cd / \
+    && rm -rf /tmp/*
+
+ENV PATH /opt/miniconda3/bin:$PATH
+
+RUN pip install nnabla-ext-cuda100-nccl2-ubuntu16
diff --git a/docker/py37/cuda100/Dockerfile b/docker/py37/cuda100/Dockerfile
@@ -0,0 +1,23 @@
+FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN umask 0 \
+    && mkdir -p /tmp/deps \
+    && cd /tmp/deps \
+    && curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+    && rm -rf Miniconda3-latest-Linux-x86_64.sh \
+    && PATH=/opt/miniconda3/bin:$PATH \
+    && conda install python=3.7 \
+    && conda update -y --all \
+    && conda install pip wheel opencv \
+    && conda clean -y --all \
+    && cd / \
+    && rm -rf /tmp/*
+
+ENV PATH /opt/miniconda3/bin:$PATH
+
+RUN pip install nnabla-ext-cuda100
diff --git a/docker/py37/cuda80/Dockerfile b/docker/py37/cuda80/Dockerfile
@@ -0,0 +1,23 @@
+FROM nvidia/cuda:8.0-cudnn7-runtime-ubuntu16.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN umask 0 \
+    && mkdir -p /tmp/deps \
+    && cd /tmp/deps \
+    && curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+    && rm -rf Miniconda3-latest-Linux-x86_64.sh \
+    && PATH=/opt/miniconda3/bin:$PATH \
+    && conda install python=3.7 \
+    && conda update -y --all \
+    && conda install pip wheel opencv \
+    && conda clean -y --all \
+    && cd / \
+    && rm -rf /tmp/*
+
+ENV PATH /opt/miniconda3/bin:$PATH
+
+RUN pip install nnabla-ext-cuda80
diff --git a/docker/py37/cuda90-multi-gpu/Dockerfile b/docker/py37/cuda90-multi-gpu/Dockerfile
@@ -0,0 +1,23 @@
+FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN umask 0 \
+    && mkdir -p /tmp/deps \
+    && cd /tmp/deps \
+    && curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+    && rm -rf Miniconda3-latest-Linux-x86_64.sh \
+    && PATH=/opt/miniconda3/bin:$PATH \
+    && conda install python=3.7 \
+    && conda update -y --all \
+    && conda install pip wheel opencv \
+    && conda clean -y --all \
+    && cd / \
+    && rm -rf /tmp/*
+
+ENV PATH /opt/miniconda3/bin:$PATH
+
+RUN pip install nnabla-ext-cuda90-nccl2-ubuntu16
diff --git a/docker/py37/cuda90/Dockerfile b/docker/py37/cuda90/Dockerfile
@@ -0,0 +1,23 @@
+FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN umask 0 \
+    && mkdir -p /tmp/deps \
+    && cd /tmp/deps \
+    && curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+    && rm -rf Miniconda3-latest-Linux-x86_64.sh \
+    && PATH=/opt/miniconda3/bin:$PATH \
+    && conda install python=3.7 \
+    && conda update -y --all \
+    && conda install pip wheel opencv \
+    && conda clean -y --all \
+    && cd / \
+    && rm -rf /tmp/*
+
+ENV PATH /opt/miniconda3/bin:$PATH
+
+RUN pip install nnabla-ext-cuda90
diff --git a/docker/py37/cuda92-multi-gpu/Dockerfile b/docker/py37/cuda92-multi-gpu/Dockerfile
@@ -0,0 +1,23 @@
+FROM nvidia/cuda:9.2-cudnn7-runtime-ubuntu16.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl bzip2 libopenmpi-dev openmpi-bin \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN umask 0 \
+    && mkdir -p /tmp/deps \
+    && cd /tmp/deps \
+    && curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+    && rm -rf Miniconda3-latest-Linux-x86_64.sh \
+    && PATH=/opt/miniconda3/bin:$PATH \
+    && conda install python=3.7 \
+    && conda update -y --all \
+    && conda install pip wheel opencv \
+    && conda clean -y --all \
+    && cd / \
+    && rm -rf /tmp/*
+
+ENV PATH /opt/miniconda3/bin:$PATH
+
+RUN pip install nnabla-ext-cuda92-nccl2-ubuntu16