Skip to content

Commit

Permalink
refine cub protobuf build dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
fanliwen committed Mar 2, 2019
1 parent 4f79e36 commit 8d0e260
Show file tree
Hide file tree
Showing 228 changed files with 221 additions and 153,048 deletions.
66 changes: 44 additions & 22 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,40 +7,62 @@ set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/)

find_package(Threads REQUIRED)

find_package(CUDA)
if (CUDA_FOUND)
add_definitions(-DHAVE_CUDA)
endif()

# OpenMP Support
find_package(OpenMP)
if (OPENMP_FOUND)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
endif()

# use cmake internal CUDA_ARCH_NAME switch
# e.g. CUDA_ARCH_NAME="Auto" will autodetect
# CUDA_ARCH_NAME="All" will use all arches
# cuda_select_nvcc_arch_flags(NVCC_ARCH_FLAGS ${CUDA_ARCH_NAME})
# list(APPEND CUDA_NVCC_FLAGS ${NVCC_ARCH_FLAGS})
# message(STATUS "Using CUDA arch flags: ${NVCC_ARCH_FLAGS_readable}")
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_60,code=sm_60)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_61,code=sm_61)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_62,code=sm_62)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_70,code=sm_70)
include(cub)
include(protobuf)

set(cuBERT_EXTERNAL_LIBRARIES
${protobuf_STATIC_LIBRARIES})
set(cuBERT_EXTERNAL_DEPENDENCIES
cub
protobuf)

set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--expt-relaxed-constexpr)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-O3;-use_fast_math;-Xptxas -O3;-Xcompiler -O3) # Flush denormals to zero
include_directories(
# Source and generated code.
src
${CMAKE_CURRENT_BINARY_DIR}
# External dependencies.
${cub_INCLUDE_DIR}
${PROTOBUF_INCLUDE_DIRS})

# MKL Support
include(mkl)
list(APPEND cuBERT_EXTERNAL_LIBRARIES ${mkl_SHARED_LIBRARIES})
list(APPEND cuBERT_EXTERNAL_DEPENDENCIES mkl_copy_shared_to_destination)
include_directories(${mkl_INCLUDE_DIRS})

include_directories(src external ${mkl_INCLUDE_DIRS})
find_package(CUDA)
if (CUDA_FOUND)
add_definitions(-DHAVE_CUDA)

# use cmake internal CUDA_ARCH_NAME switch
# e.g. CUDA_ARCH_NAME="Auto" will autodetect
# CUDA_ARCH_NAME="All" will use all arches
# cuda_select_nvcc_arch_flags(NVCC_ARCH_FLAGS ${CUDA_ARCH_NAME})
# list(APPEND CUDA_NVCC_FLAGS ${NVCC_ARCH_FLAGS})
# message(STATUS "Using CUDA arch flags: ${NVCC_ARCH_FLAGS_readable}")

set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_60,code=sm_60)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_61,code=sm_61)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_62,code=sm_62)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_70,code=sm_70)

set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--expt-relaxed-constexpr)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-O3;-use_fast_math;-Xptxas -O3;-Xcompiler -O3)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-ftz=true) # Flush denormals to zero

include_directories(${CUDA_INCLUDE_DIRS})
endif()

add_subdirectory(external)
# Let's get to work!
include(tf_core_framework)

set(SOURCE_FILES
cuBERT.cpp
Expand Down Expand Up @@ -69,8 +91,8 @@ if (CUDA_FOUND)
else ()
add_library(cuBERT SHARED ${SOURCE_FILES})
endif ()
add_dependencies(cuBERT mkl_copy_shared_to_destination)
target_link_libraries(cuBERT external ${mkl_SHARED_LIBRARIES})
add_dependencies(cuBERT ${cuBERT_EXTERNAL_DEPENDENCIES})
target_link_libraries(cuBERT tf_protos_cc ${cuBERT_EXTERNAL_LIBRARIES})

add_executable(cuBERT_benchmark benchmark.cpp)
target_link_libraries(cuBERT_benchmark cuBERT)
Expand Down Expand Up @@ -99,7 +121,7 @@ add_executable(cuBERT_test test/unit_test.cpp
test/cuBERT/bert/BertTest.cpp
test/cuBERT/multi/BertMTest.cpp
test/cuBERT_test.cpp)
target_link_libraries(cuBERT_test cuBERT ${googletest_STATIC_LIBRARIES} pthread)
target_link_libraries(cuBERT_test cuBERT ${googletest_STATIC_LIBRARIES})
target_include_directories(cuBERT_test PUBLIC ${googletest_INCLUDE_DIRS})
add_test(UnitTest cuBERT_test)

Expand Down
30 changes: 30 additions & 0 deletions cmake/cub.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
include (ExternalProject)

set(cub_URL https://github.com/NVlabs/cub/archive/1.8.0.zip)
set(cub_HASH SHA256=6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3)
set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive)

ExternalProject_Add(cub
PREFIX cub
URL ${cub_URL}
URL_HASH ${cub_HASH}
DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
BUILD_IN_SOURCE 1
PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/cub/CMakeLists.txt ${cub_BUILD}
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${cub_INCLUDE_DIR}/cub ${cub_ARCHIVE_DIR}/cub)
81 changes: 81 additions & 0 deletions cmake/protobuf.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
include (ExternalProject)

set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
set(PROTOBUF_URL https://github.com/google/protobuf.git)
set(PROTOBUF_TAG v3.6.0)

if(WIN32)
if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
set(protobuf_STATIC_LIBRARIES
debug ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/$(Configuration)/libprotobufd.lib
optimized ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/$(Configuration)/libprotobuf.lib)
set(PROTOBUF_PROTOC_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/$(Configuration)/protoc.exe)
else()
if(CMAKE_BUILD_TYPE EQUAL Debug)
set(protobuf_STATIC_LIBRARIES
${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/libprotobufd.lib)
else()
set(protobuf_STATIC_LIBRARIES
${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/libprotobuf.lib)
endif()
set(PROTOBUF_PROTOC_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/protoc.exe)
endif()

# This section is to make sure CONFIGURE_COMMAND use the same generator settings
set(PROTOBUF_GENERATOR_PLATFORM)
if (CMAKE_GENERATOR_PLATFORM)
set(PROTOBUF_GENERATOR_PLATFORM -A ${CMAKE_GENERATOR_PLATFORM})
endif()
set(PROTOBUF_GENERATOR_TOOLSET)
if (CMAKE_GENERATOR_TOOLSET)
set(PROTOBUF_GENERATOR_TOOLSET -T ${CMAKE_GENERATOR_TOOLSET})
endif()
set(PROTOBUF_ADDITIONAL_CMAKE_OPTIONS -Dprotobuf_MSVC_STATIC_RUNTIME:BOOL=OFF
-G${CMAKE_GENERATOR} ${PROTOBUF_GENERATOR_PLATFORM} ${PROTOBUF_GENERATOR_TOOLSET})
# End of section
else()
set(protobuf_STATIC_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/libprotobuf.a)
set(PROTOBUF_PROTOC_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/protoc)
endif()

ExternalProject_Add(protobuf
PREFIX protobuf
# DEPENDS zlib
GIT_REPOSITORY ${PROTOBUF_URL}
GIT_TAG ${PROTOBUF_TAG}
DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
BUILD_IN_SOURCE 1
BUILD_BYPRODUCTS ${PROTOBUF_PROTOC_EXECUTABLE} ${protobuf_STATIC_LIBRARIES}
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf
# SOURCE_SUBDIR cmake/ # Requires CMake 3.7, this will allow removal of CONFIGURE_COMMAND
# CONFIGURE_COMMAND resets some settings made in CMAKE_CACHE_ARGS and the generator used
CONFIGURE_COMMAND ${CMAKE_COMMAND} cmake/
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-Dprotobuf_BUILD_TESTS:BOOL=OFF
# -DZLIB_ROOT=${ZLIB_INSTALL}
${PROTOBUF_ADDITIONAL_CMAKE_OPTIONS}
INSTALL_COMMAND ""
CMAKE_CACHE_ARGS
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-Dprotobuf_BUILD_TESTS:BOOL=OFF
-Dprotobuf_MSVC_STATIC_RUNTIME:BOOL=OFF
# -DZLIB_ROOT:STRING=${ZLIB_INSTALL}
)
63 changes: 63 additions & 0 deletions cmake/tf_core_framework.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
########################################################
# RELATIVE_PROTOBUF_GENERATE_CPP function
########################################################
# A variant of PROTOBUF_GENERATE_CPP that keeps the directory hierarchy.
# ROOT_DIR must be absolute, and proto paths must be relative to ROOT_DIR.
function(RELATIVE_PROTOBUF_GENERATE_CPP SRCS HDRS ROOT_DIR)
if(NOT ARGN)
message(SEND_ERROR "Error: RELATIVE_PROTOBUF_GENERATE_CPP() called without any proto files")
return()
endif()

set(${SRCS})
set(${HDRS})
foreach(FIL ${ARGN})
set(ABS_FIL ${ROOT_DIR}/${FIL})
get_filename_component(FIL_WE ${FIL} NAME_WE)
get_filename_component(FIL_DIR ${ABS_FIL} PATH)
file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR})

list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc")
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h")

add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc"
"${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h"
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${ROOT_DIR} ${ABS_FIL} -I ${PROTOBUF_INCLUDE_DIRS}
DEPENDS ${ABS_FIL} protobuf
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM )
endforeach()

set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
endfunction()

########################################################
# tf_protos_cc library
########################################################

file(GLOB_RECURSE tf_protos_cc_srcs RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"${CMAKE_CURRENT_SOURCE_DIR}/tensorflow/core/framework/*.proto")

RELATIVE_PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS
${CMAKE_CURRENT_SOURCE_DIR} ${tf_protos_cc_srcs}
)

add_library(tf_protos_cc ${PROTO_SRCS} ${PROTO_HDRS})
59 changes: 0 additions & 59 deletions external/CMakeLists.txt

This file was deleted.

Loading

0 comments on commit 8d0e260

Please sign in to comment.