v0.6.0

CMakeLists.txt

@@ -0,0 +1,47 @@
+cmake_minimum_required(VERSION 3.8)
+
+set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+
+project(tarantella VERSION 0.6.0)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_FLAGS "-O3 -Wall -Wextra -Werror")
+
+option(LINK_IB "Defines whether to link against Infiniband drivers [default: disabled]" off)
+option(ENABLE_TESTING "Compile tests [default: disabled]" off)
+option(BUILD_DOCS "Build documentation [default: disabled]" off)
+
+set(SRC_DIR "${CMAKE_SOURCE_DIR}/src")
+set(CMAKE_BUILD_DIR "${CMAKE_SOURCE_DIR}/build")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+
+set(INSTALL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib/tarantella")
+set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin")
+
+find_package(GPI2 REQUIRED)
+find_package(pybind11 REQUIRED)
+find_package(Tensorflow REQUIRED)
+
+add_subdirectory(${SRC_DIR})
+add_subdirectory(${SRC_DIR}/gpi_comm_lib/gpi)
+add_subdirectory(${SRC_DIR}/gpi_comm_lib/collectives)
+add_subdirectory(${SRC_DIR}/gpi_comm_lib)
+add_subdirectory(${SRC_DIR}/gpi_comm_lib/tf_ops)
+
+if (BUILD_DOCS)
+  find_package(Sphinx)
+  add_subdirectory(docs)
+endif()
+
+if (ENABLE_TESTING)
+  find_package(Boost 1.61 REQUIRED COMPONENTS
+                                   unit_test_framework)
+  find_package(PythonModules REQUIRED COMPONENTS 
+                                      numpy
+                                      pytest)
+  enable_testing()
+  set(SLEEP_TIME_AFTER_TEST 4)
+  add_subdirectory(${CMAKE_SOURCE_DIR}/test)
+endif() 
+

LICENSE

@@ -0,0 +1,101 @@
+TARANTELLA END USER LICENSE AGREEMENT
+October 21, 2020
+
+PLEASE READ THIS LICENSE AGREEMENT CAREFULLY. BY USING THE SOFTWARE TARANTELLA YOU
+ACCEPT ALL TERMS OF THE LICENSE AGREEMENT. IF YOU DO NOT AGREE TO THE TERMS OF
+THIS LICENSE, DO NOT INSTALL, COPY, OR USE THE SOFTWARE.
+
+1.) DEFINITIONS
+
+1.1) LICENSOR: Fraunhofer Gesellschaft zur Foerderung der angewandten Forschung
+e.V., Hansastr. 27c, 80686 Muenchen, Germany, as legal entity of Fraunhofer-
+Institut fuer Techno- und Wirtschaftsmathematik, Fraunhofer-Platz 1,
+67663 Kaiserslautern, Germany.
+
+1.2) LICENSEE: The user of Tarantella under this License Agreement.
+
+1.3) LICENSED SOFTWARE: The Software Tarantella in source code and object code form
+including all executable programs.
+
+1.4) DOCUMENTATION: The Tarantella documentation, user's guide, e-mails and other explanatory
+materials accompanying the LICENSED SOFTWARE in printed or electronic form.
+
+2.) OWNERSHIP / INTELLECTUAL PROPERTY RIGHTS
+
+LICENSEE acknowledges that ownership and all intellectual property rights
+related to the LICENSED SOFTWARE and to the DOCUMENTATION, including patents,
+copyright, company or trade secrets remain with the LICENSOR.
+
+LICENSEE promises to keep and not to modify the copyright notices of the
+LICENSOR.
+
+3.) SCOPE OF LICENSE
+
+3.1) Provided LICENSEE accepts all terms of this License Agreement, LICENSEE
+is granted a non-exclusive, non-assignable right to use the LICENSED SOFTWARE,
+which means LICENSEE may use the software for an unrestricted number of users,
+as well as use the accompanying DOCUMENTATION by the actual number of users.
+
+3.2) Without prior written consent of LICENSOR or an authorized partner,
+LICENSEE may modify the source code and use the modified version of the LICENSED
+SOFTWARE for internal use only.
+
+3.2.1) LICENSEE must inform users of modified versions about the fact that the
+software differs from the original version.
+
+3.2.2) The LICENSED SOFTWARE and the modifications generated by LICENSEE shall
+remain the property of LICENSOR and no rights, including but not limited to the
+right to apply for industrial property rights, are granted to LICENSEE.
+
+3.3) Without prior written consent of LICENSOR or an authorized partner,
+LICENSEE may not:
+- use, copy or distribute the LICENSED SOFTWARE except as provided for under
+  sections 3.1 and 3.2.
+- provide commercial turn-key solutions based on the LICENSED SOFTWARE or
+  commercial services for the LICENSED SOFTWARE to any third party.
+- rent or lease the LICENSED SOFTWARE and DOCUMENTATION to any third party.
+- modify, adapt, or translate the LICENSED SOFTWARE for any third party.
+
+3.4) The license under this License Agreement relates to the LICENSED SOFTWARE.
+
+4.) LIMITED WARRANTY AND LIABILITY
+
+4.1) LICENSOR confirms that the LICENSED SOFTWARE has been developed without
+infringement of any rights of third parties, in particular patents, copyrights
+or other intellectual property rights of third parties. Nevertheless LICENSOR
+does not warrant that the use of the LICENSED SOFTWARE by LICENSEE does not
+infringe any third party intellectual property rights.
+
+4.2) LICENSEE is aware that there is a risk that the LICENSED SOFTWARE might
+damage the data or the computer of the LICENSEE or even other computers on the
+network in unpredictable ways. The use of the LICENSED SOFTWARE is at the
+exclusive risk of the LICENSEE. LICENSOR does not offer any warranty either
+expressed or implied and is not liable for any damages resulting from the use of
+the LICENSED SOFTWARE or DOCUMENTATION such as, but not limited to, data loss.
+
+4.3) Notwithstanding sections 4.1 and 4.2, the liability of the LICENSOR, its
+legal representatives and employees resulting from breach of duty or tort is
+restricted to damages caused intentionally or by gross negligence. In any case,
+the liability under this section is limited by typical, foreseeable, direct
+damages. The liability is unrestricted for damages of the body, life or health.
+
+5.) MISCELLANEOUS
+
+This License Agreement in English is the original one. The terms of this
+Agreement can only be modified or amended in writing. In case of interpretation
+controversies the terms of this Agreement shall prevail over the respective
+terms of any other agreements.
+
+This Agreement is construed under the Law of the Federal Republic of Germany.
+Therefore, any and all controversies resulting out of this Agreement shall be
+resolved under the Law of the Federal Republic of Germany excluding the German
+International Private Law Rules. The application of the UN-Convention of the
+International Sales of Goods (CISG) is explicitly excluded. Exclusive venue of
+jurisdiction for both parties shall be Munich, Germany.
+
+In case that one or several of the terms of this Agreement should be or become
+invalid or unenforceable, the validity of the other terms shall remain
+unaffected. In such a case, the parties shall replace the invalid or
+unenforceable condition by another legally effective provision meeting the
+purpose of the abolished provision to the greatest extent. The same applies in
+case of a gap of regulation.

README.md

@@ -0,0 +1,41 @@
+![Tarantella](docs/source/pics/tnt_logo_text.png)
+
+<br/><br/>
+
+Tarantella is an open-source, distributed Deep Learning framework built on top of TensorFlow 2,
+providing scalable Deep Neural Network training on CPU and GPU compute clusters.
+
+Tarantella is easy-to-use, allows to re-use existing TensorFlow 2/Keras models,
+and does not require any knowledge of parallel computing.
+
+
+## Goals
+
+Tarantella is designed to meet the following goals:
+
+* strong scalability
+* ease of use
+* synchronous training scheme
+* seamless integration with existing Keras models
+* support for GPU and CPU systems
+
+## Install
+
+To build Tarantella from source, the following dependencies are required:
+
+* [TensorFlow 2](https://www.tensorflow.org/install) (supported versions TF2.2, TF2.1, TF2.0)
+* [GPI-2](https://github.com/cc-hpc-itwm/GPI-2) (version 1.4.0)
+* [pybind11](https://github.com/pybind/pybind11) (from version 2.4.3)
+* C++ compiler (e.g., `gcc` from version 7.4.0)
+* CMake (from version 3.8)
+
+Detailed installation instructions can be found in the [technical docs](https://tarantella.readthedocs.io/en/latest/installation.html).
+
+## Resources
+
+* [Official website](https://www.tarantella.org)
+* [Technical documentation](https://tarantella.readthedocs.io/en/latest)
+
+## License
+
+[License](LICENSE)

cmake/FindDNNL.cmake

@@ -0,0 +1,37 @@
+# Finds Intel DNNL library
+# Martin Kuehn May 2020
+
+find_path(DNNL_INCLUDE_DIR
+          NAMES dnnl.hpp
+          PATHS ${DNNL_ROOT}
+                ENV DNNL_ROOT
+                ${DNNL_ROOT_DIR}
+                ENV DNNL_ROOT_DIR
+          PATH_SUFFIXES include
+          DOC "DNNL header files"
+)
+
+find_library(DNNL_LIBRARY dnnl
+             PATHS ${DNNL_ROOT}
+                   ENV DNNL_ROOT
+                   ${DNNL_ROOT_DIR}
+                   ENV DNNL_ROOT_DIR
+             PATH_SUFFIXES lib lib64
+             DOC "DNNL library files")
+
+#include (FindPackageHandleStandardArgs)
+find_package_handle_standard_args(DNNL
+                                  DEFAULT_MSG
+                                  DNNL_LIBRARY
+                                  DNNL_INCLUDE_DIR)
+
+mark_as_advanced(DNNL_INCLUDE_DIR DNNL_LIBRARY)
+
+set(DNNL_INCLUDE_DIRS ${DNNL_INCLUDE_DIR})
+set(DNNL_LIBRARIES ${DNNL_LIBRARY})
+
+if(DNNL_FOUND AND NOT TARGET dnnl)
+    add_library(dnnl SHARED IMPORTED GLOBAL)
+    target_include_directories(dnnl INTERFACE ${DNNL_INCLUDE_DIRS})
+    set_property(TARGET dnnl PROPERTY IMPORTED_LOCATION ${DNNL_LIBRARIES})
+endif()

cmake/FindGPI2.cmake

@@ -0,0 +1,133 @@
+
+#[=======================================================================[.rst:
+FindGPI2
+-------
+
+Finds the GPI2 library.
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module provides the following imported targets, if found:
+
+``GPI2::GPI2``
+  The GPI2 library
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This will define the following variables:
+
+``GPI2_FOUND``
+  True if the system has the GPI2 library.
+``GPI2_INCLUDE_DIRS``
+  Include directories needed to use GPI2.
+``GPI2_LIBRARIES``
+  Libraries needed to link to GPI2.
+``GPI2_DBG_LIBRARIES``
+  Libraries needed to link to the Debug version of GPI2.
+``GPI2_GASPI_RUN``
+  Path to ``gaspi_run``.
+
+Cache Variables
+^^^^^^^^^^^^^^^
+
+The following cache variables may also be set:
+
+``GPI2_INCLUDE_DIR``
+  The directory containing ``gaspi.h``.
+``GPI2_LIBRARY``
+  The path to the GPI2 library.
+
+#]=======================================================================]
+
+set(GPI2_LIBRARY_NAME "GPI2")
+set(GPI2_DBG_LIBRARY_NAME "GPI2-dbg")
+
+FIND_PROGRAM(GASPIRUN_PATH gaspi_run
+	PATHS
+      $ENV{PATH}
+      $ENV{LIB_DIR}/bin
+      /usr/local/bin/
+      /usr/bin/
+      )
+
+IF (GASPIRUN_PATH) 
+      get_filename_component(GASPIRUN_FOUND_HOME ${GASPIRUN_PATH} DIRECTORY)
+      get_filename_component(GPI2_INSTALLED_PATH ${GASPIRUN_FOUND_HOME} DIRECTORY)
+      get_filename_component(GPI2_INSTALLED_PATH ${GPI2_INSTALLED_PATH} REALPATH)
+ENDIF(GASPIRUN_PATH) 
+
+find_path (GPI2_INCLUDE_DIR GASPI.h
+              PATHS ${GPI2_DEFAULT_PATH} ${GPI2_INSTALLED_PATH}
+              PATHS ENV LD_LIBRARY_PATH DYLD_LIBRARY_PATH
+              PATH_SUFFIXES include)
+
+find_library (GPI2_DBG_LIBRARY ${GPI2_DBG_LIBRARY_NAME}
+              PATHS ${GPI2_DEFAULT_PATH} ${GPI2_INSTALLED_PATH}
+              PATHS ENV LD_LIBRARY_PATH DYLD_LIBRARY_PATH
+              PATH_SUFFIXES lib lib64)
+
+find_library (GPI2_LIBRARY ${GPI2_LIBRARY_NAME}
+              PATHS ${GPI2_DEFAULT_PATH} ${GPI2_INSTALLED_PATH}
+              PATHS ENV LD_LIBRARY_PATH DYLD_LIBRARY_PATH
+              PATH_SUFFIXES lib lib64)
+
+if (GPI2_DBG_LIBRARY)
+    message(STATUS "GPI2-dbg library path: ${GPI2_DBG_LIBRARY}" )       
+else(GPI2_DBG_LIBRARY)
+    message(STATUS "GPI2-dbg  library path: not found" )
+endif()
+
+
+if (GPI2_LIBRARY)
+    message(STATUS "GPI2 library path: ${GPI2_LIBRARY}" )    
+else(GPI2_LIBRARY)
+    message(STATUS "GPI2 library path: not found" )
+endif()
+
+
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set GPI2_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args(GPI2 DEFAULT_MSG
+                                  GASPIRUN_PATH
+                                  GPI2_DBG_LIBRARY GPI2_LIBRARY)
+
+mark_as_advanced(GPI2_INCLUDE_DIR GASPIRUN_PATH
+                 GPI2_DBG_LIBRARY GPI2_LIBRARY)
+set(GPI2_INCLUDE_DIRS ${GPI2_INCLUDE_DIR} )
+set(GPI2_DBG_LIBRARIES ${GPI2_DBG_LIBRARY} )
+set(GPI2_LIBRARIES ${GPI2_LIBRARY} )
+set(GPI2_GASPI_RUN ${GASPIRUN_PATH})
+
+message(STATUS "Found GPI2: " ${GPI2_FOUND})
+
+if(GPI2_FOUND AND NOT TARGET GPI2::GPI2)
+    set(THREADS_PREFER_PTHREAD_FLAG ON)
+    find_package(Threads REQUIRED)
+    add_library(GPI2::GPI2 SHARED IMPORTED GLOBAL)
+    target_link_libraries(GPI2::GPI2 INTERFACE Threads::Threads)
+    target_include_directories(GPI2::GPI2 INTERFACE ${GPI2_INCLUDE_DIRS})
+    set_property(TARGET GPI2::GPI2 PROPERTY IMPORTED_LOCATION ${GPI2_LIBRARIES})
+
+    add_library(GPI2::GPI2dbg SHARED IMPORTED GLOBAL)
+    target_link_libraries(GPI2::GPI2dbg INTERFACE Threads::Threads)
+    target_include_directories(GPI2::GPI2dbg INTERFACE ${GPI2_INCLUDE_DIRS})
+    set_property(TARGET GPI2::GPI2dbg  PROPERTY IMPORTED_LOCATION ${GPI2_DBG_LIBRARIES})
+
+    if (LINK_IB)
+        find_package(IBverbs)
+
+        if (IBverbs_FOUND)
+            message (STATUS "GPI2: linking against ibverbs")
+            target_link_libraries(GPI2::GPI2 INTERFACE IBverbs::IBverbs)
+            target_link_libraries(GPI2::GPI2dbg INTERFACE IBverbs::IBverbs)
+        else()
+            message (FATAL_ERROR "GPI2: could not find ibverbs, disable Infiniband \
+                                  support (-DLINK_IB=OFF) to load GPI-2")
+        endif()
+    else()
+        message (STATUS "GPI2: loading library without Infiniband support")
+    endif()
+endif()