Skip to content

Commit

Permalink
Merge pull request #182 from sony/feature/20190814-include-cuda-lib-i…
Browse files Browse the repository at this point in the history 
…n-wheel

Includes cuda lib in wheel
  • Loading branch information
@YasunariZHashimoto
YasunariZHashimoto committed Sep 4, 2019
2 parents 8e6c069 + d704179 commit a291d2a
Show file tree
Hide file tree
Showing 8 changed files with 156 additions and 44 deletions.
27 changes: 14 additions & 13 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,21 @@ endif()
include(${PROJECT_SOURCE_DIR}/build-tools/cmake/get_cuda_version.cmake)

set(CPACK_PACKAGE_NAME "nnabla-cpplib-cuda${CUDA_VERSION}-cudnn${CUDNN_VERSION}")
find_program(LSB_RELEASE lsb_release)
message("LSB_RELEASE" ${LSB_RELEASE})
if(EXISTS ${LSB_RELEASE})

execute_process(COMMAND ${LSB_RELEASE} -si
OUTPUT_VARIABLE LSB_RELEASE_ID
OUTPUT_STRIP_TRAILING_WHITESPACE
)
execute_process(COMMAND ${LSB_RELEASE} -sr
OUTPUT_VARIABLE LSB_RELEASE_RELEASE
OUTPUT_STRIP_TRAILING_WHITESPACE
)
set(CPACK_SYSTEM_NAME ${LSB_RELEASE_ID}${LSB_RELEASE_RELEASE})
if(UNIX)
find_program(LSB_RELEASE lsb_release)
if(EXISTS ${LSB_RELEASE})
execute_process(COMMAND ${LSB_RELEASE} -si
OUTPUT_VARIABLE LSB_RELEASE_ID
OUTPUT_STRIP_TRAILING_WHITESPACE
)
execute_process(COMMAND ${LSB_RELEASE} -sr
OUTPUT_VARIABLE LSB_RELEASE_RELEASE
OUTPUT_STRIP_TRAILING_WHITESPACE
)
set(CPACK_SYSTEM_NAME ${LSB_RELEASE_ID}${LSB_RELEASE_RELEASE})
endif()
endif()

set(CPACK_PACKAGE_VERSION_MAJOR ${version_major})
set(CPACK_PACKAGE_VERSION_MINOR ${version_minor})
set(CPACK_PACKAGE_VERSION_PATCH ${version_patch})
Expand Down
9 changes: 6 additions & 3 deletions build-tools/make/build-with-docker.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ DOCKER_RUN_OPTS += -e NNABLA_DIRECTORY=$(NNABLA_DIRECTORY)
NNABLA_EXT_CUDA_DIRECTORY ?= $(shell pwd)
DOCKER_RUN_OPTS += -e NNABLA_EXT_CUDA_DIRECTORY=$(NNABLA_EXT_CUDA_DIRECTORY)
DOCKER_RUN_OPTS += -e CMAKE_OPTS=$(CMAKE_OPTS)
DOCKER_RUN_OPTS += -e INCLUDE_CUDA_CUDNN_LIB_IN_WHL=$(INCLUDE_CUDA_CUDNN_LIB_IN_WHL)

include $(NNABLA_EXT_CUDA_DIRECTORY)/build-tools/make/options.mk
ifndef NNABLA_BUILD_INCLUDED
Expand All @@ -35,6 +36,8 @@ ifndef NNABLA_BUILD_WITH_DOCKER_INCLUDED
include $(NNABLA_DIRECTORY)/build-tools/make/build-with-docker.mk
endif

NVIDIA_DOCKER_WRAPPER=$(NNABLA_EXT_CUDA_DIRECTORY)/build-tools/scripts/nvidia-docker.sh

CUDA_SUFFIX = $(CUDA_VERSION_MAJOR)$(CUDA_VERSION_MINOR)-cudnn$(CUDNN_VERSION)

DOCKER_IMAGE_ID_BUILD_NNABLA_EXT_CUDA = $(shell md5sum $(NNABLA_EXT_CUDA_DIRECTORY)/docker/development/Dockerfile.build$(ARCH_SUFFIX) |cut -d \ -f 1)
Expand Down Expand Up @@ -147,17 +150,17 @@ bwd-nnabla-ext-cuda-wheel-multi-gpu-only: docker_image_build_cuda_multi_gpu
.PHONY: bwd-nnabla-ext-cuda-test
bwd-nnabla-ext-cuda-test: docker_image_build_cuda
cd $(NNABLA_EXT_CUDA_DIRECTORY) \
&& nvidia-docker run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_BUILD_NNABLA_EXT_CUDA) make -f build-tools/make/build.mk nnabla-ext-cuda-test-local
&& ${NVIDIA_DOCKER_WRAPPER} run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_BUILD_NNABLA_EXT_CUDA) make -f build-tools/make/build.mk nnabla-ext-cuda-test-local

.PHONY: bwd-nnabla-ext-cuda-multi-gpu-test
bwd-nnabla-ext-cuda-multi-gpu-test: docker_image_build_cuda_multi_gpu
cd $(NNABLA_EXT_CUDA_DIRECTORY) \
&& nvidia-docker run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_BUILD_NNABLA_EXT_CUDA_MULTI_GPU) make -f build-tools/make/build.mk nnabla-ext-cuda-multi-gpu-test-local
&& ${NVIDIA_DOCKER_WRAPPER} run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_BUILD_NNABLA_EXT_CUDA_MULTI_GPU) make -f build-tools/make/build.mk nnabla-ext-cuda-multi-gpu-test-local

.PHONY: bwd-nnabla-ext-cuda-shell
bwd-nnabla-ext-cuda-shell: docker_image_build_cuda
cd $(NNABLA_EXT_CUDA_DIRECTORY) \
&& nvidia-docker run $(DOCKER_RUN_OPTS) -it --rm ${DOCKER_IMAGE_BUILD_NNABLA_EXT_CUDA} make nnabla-ext-cuda-shell
&& ${NVIDIA_DOCKER_WRAPPER} run $(DOCKER_RUN_OPTS) -it --rm ${DOCKER_IMAGE_BUILD_NNABLA_EXT_CUDA} make nnabla-ext-cuda-shell

########################################################################################################################
# Docker image with current nnabla
Expand Down
4 changes: 2 additions & 2 deletions build-tools/make/options.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ DOCKER_RUN_OPTS += -e CUDA_VERSION_MINOR=$(CUDA_VERSION_MINOR)
export CUDNN_VERSION ?= 7
DOCKER_RUN_OPTS += -e CUDNN_VERSION=$(CUDNN_VERSION)

export WHL_NO_PREFIX ?= False
DOCKER_RUN_OPTS += -e WHL_NO_PREFIX=$(WHL_NO_PREFIX)
export WHL_NO_CUDA_SUFFIX ?= False
DOCKER_RUN_OPTS += -e WHL_NO_CUDA_SUFFIX=$(WHL_NO_CUDA_SUFFIX)

ifndef NNABLA_OPTIONS_INCLUDED
include $(NNABLA_DIRECTORY)/build-tools/make/options.mk
Expand Down
19 changes: 19 additions & 0 deletions build-tools/scripts/nvidia-docker.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/bash

if which nvidia-docker >/dev/null
then
nvidia-docker "$@"
else
if [ "$1" == "run" ]
then
GPU=\"device=${NV_GPU}\"
if [ "${NV_GPU}" == "" ]
then
GPU=all
fi
docker run --gpus=$GPU "${@:2}"
else
docker "$@"
fi
fi

3 changes: 3 additions & 0 deletions python/setup.cfg.build-wheel.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,6 @@ cpu_target_file=${CPPLIB_LIBRARY}
cuda_target_name=${CPPLIB_CUDA_LIBRARY_NAME}
cuda_target_file_name=${CPPLIB_CUDA_TARGET_FILE_NAME}
cuda_target_file=${CPPLIB_CUDA_TARGET_FILE}

cuda_toolkit_root_dir=${CUDA_TOOLKIT_ROOT_DIR}
cudnn_include_dir=${CUDNN_INCLUDE_DIR}
3 changes: 3 additions & 0 deletions python/setup.cfg.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,6 @@ cpu_target_file=${CPPLIB_LIBRARY}
cuda_target_name=${NBLA_CUDA_LIBRARY_NAME}
cuda_target_file_name=$<TARGET_FILE_NAME:${NBLA_CUDA_LIBRARY_NAME}>
cuda_target_file=$<TARGET_FILE:${NBLA_CUDA_LIBRARY_NAME}>

cuda_toolkit_root_dir=${CUDA_TOOLKIT_ROOT_DIR}
cudnn_include_dir=${CUDNN_INCLUDE_DIR}
80 changes: 67 additions & 13 deletions python/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from collections import namedtuple
import copy
import shutil
import subprocess
import sys

root_dir = realpath(dirname(__file__))
Expand Down Expand Up @@ -89,7 +90,15 @@ def get_libinfo():
print("CUDA Library name:", cuda_lib.name)
print("CUDA Library file name:", cuda_lib.file_name)
print("CUDA Library file:", cuda_lib.path)
return cpu_lib, cuda_lib

if 'INCLUDE_CUDA_CUDNN_LIB_IN_WHL' in os.environ and os.environ['INCLUDE_CUDA_CUDNN_LIB_IN_WHL'] == 'True':
print("CUDA/cuDNN libraries will include into wheel package.")
libs = [cfgp.get("cmake", "cuda_toolkit_root_dir"),
os.path.dirname(cfgp.get("cmake", "cudnn_include_dir"))]
else:
libs = None

return cpu_lib, cuda_lib, libs


def get_cpu_extopts(lib):
Expand All @@ -114,7 +123,7 @@ def get_cpu_extopts(lib):
return ext_opts


def cuda_config(root_dir, cuda_lib, ext_opts):
def cuda_config(root_dir, cuda_lib, ext_opts, lib_dirs):
# With CUDA
src_dir = join(root_dir, 'src')
path_cuda_pkg = join(src_dir, 'nnabla_ext', 'cuda')
Expand All @@ -126,14 +135,59 @@ def cuda_config(root_dir, cuda_lib, ext_opts):
shutil.copyfile(cuda_lib.path, cuda_lib_out)
package_data = {cuda_pkg: [cuda_lib.file_name]}

if sys.platform == 'win32':
libdir = dirname(cuda_lib.path)
libname, _ = splitext(cuda_lib.file_name)
cuda_ext_lib_file_name = libname + '.lib'
cuda_ext_lib_path = join(libdir, cuda_ext_lib_file_name)
cuda_ext_lib_out = join(path_cuda_pkg, cuda_ext_lib_file_name)
shutil.copyfile(cuda_ext_lib_path, cuda_ext_lib_out)
package_data[cuda_pkg].append(cuda_ext_lib_file_name)
if lib_dirs is not None:
if sys.platform.startswith('linux'):
out = subprocess.check_output(['ldd', cuda_lib_out])
for l in out.splitlines():
ls = l.strip().decode('ascii').split()
if len(ls) >= 3:
libname = ls[0]
libfile = ls[2]

# Copy libraries into WHL file.
# libcu* : CUDA/cuDNN
# libnccl : NCCL2
if libname.startswith('libcu') or \
libname.startswith('libnccl'):
print('Copying {}'.format(libname))
path_out = join(path_cuda_pkg, libname)
shutil.copyfile(libfile, path_out)
package_data[cuda_pkg].append(libname)

elif sys.platform == 'win32':
libdir = dirname(cuda_lib.path)
libname, _ = splitext(cuda_lib.file_name)
cuda_ext_lib_file_name = libname + '.lib'
cuda_ext_lib_path = join(libdir, cuda_ext_lib_file_name)
cuda_ext_lib_out = join(path_cuda_pkg, cuda_ext_lib_file_name)
shutil.copyfile(cuda_ext_lib_path, cuda_ext_lib_out)
package_data[cuda_pkg].append(cuda_ext_lib_file_name)

def search_dependencies(lib, libs=[]):
print('Searching libs in {}'.format(lib))
out = subprocess.check_output(['dumpbin', '/DEPENDENTS', lib])
for l in out.splitlines():
l = l.strip().decode('ascii')
if l[:2] == 'cu' and l[-4:] == '.dll':
copied = False
for d in lib_dirs:
for currentdir, dirs, files in os.walk(d):
if l in files and not copied and l not in libs:
print('Copying {}'.format(l))
path_in = join(currentdir, l)
libs = search_dependencies(path_in, libs)
path_out = join(path_cuda_pkg, l)
shutil.copyfile(path_in, path_out)
libs.append(l)
copied = True
if not copied:
print('Shared library {} is not found.'.format(l))
sys.exit(-1)
return libs

for d in search_dependencies(cuda_lib_out):
print('LIB: {}'.format(d))
package_data[cuda_pkg].append(d)

cuda_ext_opts = copy.deepcopy(ext_opts)
cuda_ext_opts['libraries'] += [cuda_lib.name]
Expand Down Expand Up @@ -166,14 +220,14 @@ def cudnn_config(root_dir, cuda_lib, cuda_ext_opts):


def get_setup_config(root_dir):
cpu_lib, cuda_lib = get_libinfo()
cpu_lib, cuda_lib, libs = get_libinfo()

packages = ['nnabla_ext']
package_dir = {'nnabla_ext': join(root_dir, 'src', 'nnabla_ext')}
package_data = {}
ext_modules = []

cuda_ext = cuda_config(root_dir, cuda_lib, get_cpu_extopts(cpu_lib))
cuda_ext = cuda_config(root_dir, cuda_lib, get_cpu_extopts(cpu_lib), libs)
packages += cuda_ext.packages
package_dir.update(cuda_ext.package_dir)
package_data.update(cuda_ext.package_data)
Expand All @@ -186,7 +240,7 @@ def get_setup_config(root_dir):
ext_modules += cudnn_ext.ext_modules

cuda_version = ''
if 'WHL_NO_PREFIX' in os.environ and os.environ['WHL_NO_PREFIX'] == 'True':
if 'WHL_NO_CUDA_SUFFIX' in os.environ and os.environ['WHL_NO_CUDA_SUFFIX'] == 'True':
cuda_version = ''
elif 'CUDA_VERSION_MAJOR' in os.environ:
cuda_version = os.environ['CUDA_VERSION_MAJOR'] + \
Expand Down
55 changes: 42 additions & 13 deletions python/src/nnabla_ext/cuda/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,48 @@
__email__
)

try:
from .init import (
clear_memory_cache,
array_classes,
device_synchronize,
get_device_count,
get_devices,
StreamEventHandler)
except:
print('Please install CUDA version {}.'.format(__cuda_version__))
print(' and CUDNN version {}.'.format(__cudnn_version__))
print('Or install correct nnabla_ext_cuda for installed version of CUDA/CUDNN.')
raise

#
# Workaround for loading shared library.
#
MAX_RETRY_LOAD_SHARED_LIB = 100


def load_shared_from_error(err):
import ctypes
import os
import sys
base = os.path.dirname(__file__)
es = str(err).split(':')
if len(es) > 0:
fn = os.path.join(base, es[0])
if os.path.exists(fn):
retry = 0
while retry < MAX_RETRY_LOAD_SHARED_LIB:
retry += 1
try:
ctypes.cdll.LoadLibrary(fn)
retry = MAX_RETRY_LOAD_SHARED_LIB
except OSError as err:
load_shared_from_error(err)
else:
raise err


retry = 0
while retry < MAX_RETRY_LOAD_SHARED_LIB:
retry += 1
try:
from .init import (
clear_memory_cache,
array_classes,
device_synchronize,
get_device_count,
get_devices,
StreamEventHandler)
retry = MAX_RETRY_LOAD_SHARED_LIB
except ImportError as err:
load_shared_from_error(err)

from nnabla.variable import Context

Expand Down

0 comments on commit a291d2a

Please sign in to comment.