Merge branch 'master' into feature/20180401-file-format-converter

CMakeLists.txt

@@ -279,10 +279,12 @@ if(BUILD_PYTHON_PACKAGE)
       endif()
     endif()
 
+    set(NBLA_LIBRARY_NAME ${CPPLIB_LIBRARY})  # No one sets if !BUILD_CPP_LIB
     set(CPPLIB_LIBRARY_NAME "nnabla")
     set(CPPLIB_TARGET_FILE ${CPPLIB_LIBRARY})
     get_filename_component(CPPLIB_TARGET_FILE_NAME ${CPPLIB_TARGET_FILE} NAME)
-
+
+    # TODO: Debug mode in windows
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/python/setup.cfg.build-wheel.in
       ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.cfg)
   endif()
@@ -305,7 +307,7 @@ if(BUILD_PYTHON_PACKAGE)
   set(NBLA_PYTHON_SETUP ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py)
   set(NBLA_PYTHON_OUTPUT ${CMAKE_BINARY_DIR}/build)
   set(NBLA_PYTHON_OUTPUT_WHEEL ${NBLA_PYTHON_OUTPUT}/.timestamp.wheel)
-
+  set(NBLA_WHEEL_DEPENDS ${NBLA_PYTHON_DEPS} ${NBLA_LIBRARY_NAME})
   # Wheel
   if(UNIX)
     if(MAKE_MANYLINUX_WHEEL)
@@ -316,28 +318,27 @@ if(BUILD_PYTHON_PACKAGE)
         COMMAND ${CMAKE_SOURCE_DIR}/build-tools/auditwheel-nnabla ${CMAKE_BINARY_DIR}/dist/*-linux_*.whl
         COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_BINARY_DIR}/dist/*-linux*.whl
         COMMAND ${CMAKE_COMMAND} -E touch ${NBLA_PYTHON_OUTPUT_WHEEL}
-        DEPENDS ${NBLA_PYTHON_DEPS} ${NBLA_LIBRARY_NAME} ${NBLA_PYTHON_OUTPUT_BUILD})
+        DEPENDS ${NBLA_WHEEL_DEPENDS})
     else()
       add_custom_command(OUTPUT ${NBLA_PYTHON_OUTPUT_WHEEL}
         COMMAND ${CMAKE_COMMAND} -E remove -f ${NBLA_PYTHON_OUTPUT_CYTHON}
         COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_BINARY_DIR}/dist/*.whl
         COMMAND ${PYTHON_COMMAND} ${NBLA_PYTHON_SETUP} -q bdist_wheel
         COMMAND ${CMAKE_COMMAND} -E touch ${NBLA_PYTHON_OUTPUT_WHEEL}
-        DEPENDS ${NBLA_PYTHON_DEPS} ${NBLA_LIBRARY_NAME} ${NBLA_PYTHON_OUTPUT_BUILD})
+        DEPENDS ${NBLA_WHEEL_DEPENDS})
     endif()
   else()
     add_custom_command(OUTPUT ${NBLA_PYTHON_OUTPUT_WHEEL}
       COMMAND ${CMAKE_COMMAND} -E remove -f ${NBLA_PYTHON_OUTPUT_CYTHON}
       COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_BINARY_DIR}/dist/*.whl
       COMMAND ${PYTHON_COMMAND} ${NBLA_PYTHON_SETUP} -q bdist_wheel
       COMMAND ${CMAKE_COMMAND} -E touch ${NBLA_PYTHON_OUTPUT_WHEEL}
-      DEPENDS ${NBLA_PYTHON_DEPS} ${NBLA_LIBRARY_NAME} ${NBLA_PYTHON_OUTPUT_BUILD})
+      DEPENDS ${NBLA_WHEEL_DEPENDS})
   endif()
   if(BUILD_CPP_LIB)
-    add_custom_target(wheel ALL DEPENDS ${NBLA_PYTHON_OUTPUT_WHEEL} ${NBLA_PYTHON_OUTPUT_BUILD})
-    add_dependencies(wheel ${NBLA_LIBRARY_NAME})
+    add_custom_target(wheel ALL DEPENDS ${NBLA_PYTHON_OUTPUT_WHEEL})
   else()
-    add_custom_target(wheel DEPENDS ${NBLA_PYTHON_OUTPUT_WHEEL} ${NBLA_PYTHON_OUTPUT_BUILD})
+    add_custom_target(wheel DEPENDS ${NBLA_PYTHON_OUTPUT_WHEEL})
   endif()
 
   if(UNIX OR APPLE)

build-tools/make/build-with-docker.mk

@@ -39,6 +39,7 @@ DOCKER_RUN_OPTS += -v $$(pwd):$$(pwd)
 DOCKER_RUN_OPTS += -w $$(pwd)
 DOCKER_RUN_OPTS += -u $$(id -u):$$(id -g)
 DOCKER_RUN_OPTS += -e HOME=/tmp
+DOCKER_RUN_OPTS += -e CMAKE_OPTS=$(CMAKE_OPTS)
 
 ########################################################################################################################
 # Docker images

build-tools/make/build.mk

@@ -35,7 +35,15 @@ nnabla-clean-all:
 # Auto Format
 .PHONY: nnabla-auto-format
 nnabla-auto-format:
-	python $(NNABLA_DIRECTORY)/build-tools/auto_format .
+	python $(NNABLA_DIRECTORY)/build-tools/auto_format . --exclude \
+		'\./src/nbla/(function|solver)/\w+\.cpp' \
+		'\./src/nbla/init.cpp' \
+		'\./python/src/nnabla/\w+\.(cpp|hpp|h|c)' \
+		'\./python/src/nnabla/(solver.pyx|function.pyx|function.pxd|function_bases.py)' \
+		'\./python/src/nnabla/utils/(save|load)_function.py' \
+		'\./src/nbla_utils/nnp_impl_create_function.cpp' \
+		'\./src/nbla_utils/nnabla\.pb\.(h|cc)'
+
 
 ########################################################################################################################
 # Doc
@@ -46,6 +54,7 @@ nnabla-doc:
 	&& cmake -DBUILD_CPP_LIB=ON \
 		-DBUILD_CPP_UTILS=OFF \
 		-DBUILD_PYTHON_PACKAGE=ON \
+		$(CMAKE_OPTS) \
 		$(NNABLA_DIRECTORY)
 	make -C $(DOC_DIRECTORY) -j$(PARALLEL_BUILD_NUM) all wheel doc
 
@@ -64,6 +73,7 @@ nnabla-cpplib:
 		-DNNABLA_UTILS_STATIC_LINK_DEPS=$(NNABLA_UTILS_STATIC_LINK_DEPS) \
 		-DNNABLA_UTILS_WITH_HDF5=$(NNABLA_UTILS_WITH_HDF5) \
 		-DBUILD_PYTHON_PACKAGE=OFF \
+		$(CMAKE_OPTS) \
 		$(NNABLA_DIRECTORY)
 	@$(MAKE) -C $(BUILD_DIRECTORY_CPPLIB) -j$(PARALLEL_BUILD_NUM)
 
@@ -80,6 +90,7 @@ nnabla-wheel:
 		-DMAKE_MANYLINUX_WHEEL=$(MAKE_MANYLINUX_WHEEL) \
 		-DCPPLIB_BUILD_DIR=$(BUILD_DIRECTORY_CPPLIB) \
 		-DCPPLIB_LIBRARY=$(BUILD_DIRECTORY_CPPLIB)/lib/libnnabla.so \
+		$(CMAKE_OPTS) \
 		$(NNABLA_DIRECTORY)
 	@$(MAKE) -C $(BUILD_DIRECTORY_WHEEL) wheel
 

include/nbla/communicator.hpp

@@ -17,6 +17,7 @@
 #ifndef __NBLA_COMMUNICATOR_HPP__
 #define __NBLA_COMMUNICATOR_HPP__
 #include <nbla/array.hpp>
+#include <nbla/computation_graph/variable.hpp>
 #include <nbla/context.hpp>
 #include <nbla/variable.hpp>
 
@@ -164,10 +165,30 @@ class NBLA_API Communicator {
   virtual void all_reduce(NdArrayPtr ndarray, bool division = false,
                           bool inplace = false, const string &group = "world");
 
-  /** reduce_scatter.
+  /** all_reduce over parameters added.
+
+  @param ndarray_list Vector of NdArrayPtr
+  @param pack_size The number of values contained in the packed data.
+  @param division Divide the reduced value.
+   */
+  virtual CommunicatorBackwardCallbackPtr
+  all_reduce_callback(const vector<NdArrayPtr> &ndarray_list, size_t pack_size,
+                      bool division = false, const string &group = "world");
+
+  /** all_reduce over parameters added.
+
+  @param ndarray NdArrayPtr
+  @param pack_size The number of values contained in the packed data.
+  @param division Divide the reduced value.
+   */
+  virtual CommunicatorBackwardCallbackPtr
+  all_reduce_callback(NdArrayPtr ndarray, size_t pack_size,
+                      bool division = false, const string &group = "world");
+
+  /** reducescatter.
 
-         @param ndarray_list Vector of NdArrayPtr
-         @param ndarray NdArrayPtr
+   @param ndarray_list Vector of NdArrayPtr
+   @param ndarray NdArrayPtr
    @param division Divide the reduced value.
    @param group Name of a group.
    */

include/nbla/computation_graph/variable.hpp

@@ -15,14 +15,26 @@
 #ifndef __NBLA_COMPUTATION_GRAPH_VARIABLE_HPP__
 #define __NBLA_COMPUTATION_GRAPH_VARIABLE_HPP__
 
+#include <nbla/function.hpp>
 #include <nbla/variable.hpp>
 
+#include <memory>
+
 namespace nbla {
 
 // Forward declaration
 class CgFunction;
 typedef shared_ptr<CgFunction> CgFunctionPtr;
 
+/** Callback functions during backward.
+ */
+struct CommunicatorBackwardCallback {
+  virtual void on_finish_function_backward(const CgFunctionPtr &ptr) {}
+  virtual void on_finish_backward() {}
+};
+typedef shared_ptr<CommunicatorBackwardCallback>
+    CommunicatorBackwardCallbackPtr;
+
 /** Computation graph variable.
 
 A Variable object is held in this object as a data container. In addition,
@@ -132,11 +144,16 @@ class CgVariable {
                  set, its gradients are set as 1.
       @param[in] clear_buffer Clears the no longer referenced variables
                  during backpropagation to save memory.
+      @param     communicator_callbacks The callback functions invoked when 1)
+                 backward computation of each function is finished and
+                 2) all backward computation is finished.
 
       @seealso set_persistent() to prevent a specific variable to be cleared
                during forward propagation.
   */
-  NBLA_API void backward(NdArrayPtr grad = nullptr, bool clear_buffer = false);
+  NBLA_API void
+  backward(NdArrayPtr grad = nullptr, bool clear_buffer = false,
+           vector<CommunicatorBackwardCallbackPtr> communicator_callbacks = {});
 
   /** @copydoc function_reference_count_
    */

python/src/nnabla/_variable.pxd

@@ -44,6 +44,9 @@ cdef extern from "nbla/variable.hpp" namespace "nbla":
 cdef extern from "nbla/computation_graph/variable.hpp" namespace "nbla":
     cdef cppclass CgFunction
     ctypedef shared_ptr[CgFunction] CgFunctionPtr
+    cdef cppclass CCommunicatorBackwardCallback "nbla::CommunicatorBackwardCallback":
+        CCommunicatorBackwardCallback() except+
+    ctypedef shared_ptr[CCommunicatorBackwardCallback] CommunicatorBackwardCallbackPtr
     cdef cppclass CgVariable:
         CgVariable(cpp_bool need_grad) except+
         CgVariable(Shape_t shape, cpp_bool need_grad) except+
@@ -54,7 +57,7 @@ cdef extern from "nbla/computation_graph/variable.hpp" namespace "nbla":
         int rank() const
         void set_rank(int rank) except+
         void forward(cpp_bool clear_buffer, cpp_bool clear_no_need_grad) nogil except+
-        void backward(NdArrayPtr grad, cpp_bool clear_buffer) nogil except+
+        void backward(NdArrayPtr grad, cpp_bool clear_buffer, vector[CommunicatorBackwardCallbackPtr] communicator_callbacks) nogil except+
         void set_persistent(cpp_bool b)
         cpp_bool persistent()
     ctypedef shared_ptr[CgVariable] CgVariablePtr
@@ -82,6 +85,12 @@ cdef class Context:
     cdef public str device_id
 
 
+cdef class CommunicatorBackwardCallback:
+    cdef CommunicatorBackwardCallbackPtr var
+
+    @staticmethod
+    cdef create_from_ccallback(CommunicatorBackwardCallbackPtr varsp)
+
 cdef class Variable:
     cdef CgVariablePtr var
     cdef CgVariable * varp

python/src/nnabla/_variable.pyx

@@ -83,6 +83,14 @@ cdef class Context:
         return repr(self)
 
 
+cdef class CommunicatorBackwardCallback:
+    @staticmethod
+    cdef create_from_ccallback(shared_ptr[CCommunicatorBackwardCallback] varsp):
+        var = CommunicatorBackwardCallback()
+        var.var = varsp
+
+        return var
+
 cdef class Variable:
     """
     :class:`nnabla.Variable` is used to construct computation graphs (neural networks) together
@@ -433,7 +441,7 @@ cdef class Variable:
         with nogil:
             self.varp.forward(clear_buffer, clear_no_need_grad)
 
-    def backward(self, grad=1, cpp_bool clear_buffer=False):
+    def backward(self, grad=1, cpp_bool clear_buffer=False, communicator_callbacks=None):
         """
         Performs a backward propagation starting from this variable until
         the root variable(s) is/are reached in the function graph.
@@ -450,6 +458,10 @@ cdef class Variable:
                 variable.
             clear_buffer(bool): Clears the no longer referenced variables
                 during backpropagation to save memory.  
+            communicator_callbacks(:obj:`nnabla.CommunicatorBackwardCallback` or list of :obj:`nnabla.CommunicatorBackwardCallback`):
+                The callback functions invoked when 1) backward computation
+                of each function is finished and 2) all backward
+                computation is finished.
 
         """
         cdef NdArrayPtr p
@@ -471,8 +483,15 @@ cdef class Variable:
             arr.data = grad
             p = ( < NdArray > arr).arr
 
+        cdef vector[CommunicatorBackwardCallbackPtr] callback_list
+        if type(communicator_callbacks) == list:
+            for x in communicator_callbacks:
+                callback_list.push_back(( < CommunicatorBackwardCallback?> x).var)
+        elif type(communicator_callbacks) != type(None):
+            callback_list.push_back(( < CommunicatorBackwardCallback?> communicator_callbacks).var)
+
         with nogil:
-            self.varp.backward(p, clear_buffer)
+            self.varp.backward(p, clear_buffer, callback_list)
 
     def unlinked(self):
         """

python/src/nnabla/communicator.pxd

@@ -19,7 +19,7 @@ from libcpp.string cimport string
 from libcpp.memory cimport shared_ptr
 from libcpp cimport bool as cpp_bool
 cimport _variable
-from _variable cimport CVariable, CContext, dtypes
+from _variable cimport CVariable, CContext, dtypes, CommunicatorBackwardCallbackPtr
 from _nd_array cimport CNdArray
 
 
@@ -47,6 +47,8 @@ cdef extern from "nbla/communicator.hpp" namespace "nbla":
         void allreduce(cpp_bool division, cpp_bool inplace) nogil except +
         void all_reduce(const vector[shared_ptr[CNdArray]] & ndarray_list, cpp_bool division, cpp_bool inplace, const string & group) nogil except +
         void all_reduce(shared_ptr[CNdArray] data, cpp_bool division, cpp_bool inplace, const string & group) nogil except +
+        CommunicatorBackwardCallbackPtr all_reduce_callback(const vector[shared_ptr[CNdArray]] & ndarray_list, size_t pack_size, cpp_bool division, const string & group) except +
+        CommunicatorBackwardCallbackPtr all_reduce_callback(shared_ptr[CNdArray] data, size_t pack_size, cpp_bool division, const string & group) except +
         void reduce_scatter(const vector[shared_ptr[CNdArray]] & ndarray_list, shared_ptr[CNdArray] ndarray, cpp_bool division, const string & group) nogil except +
         void bcast(const vector[shared_ptr[CNdArray]] & ndarray_list, int src, cpp_bool inplace, const string & group) nogil except +
         void bcast(shared_ptr[CNdArray] ndarray, int src, cpp_bool inplace, const string & group) nogil except +

python/src/nnabla/communicator.pyx

@@ -28,6 +28,7 @@ from communicator cimport CCommunicator
 
 cimport _variable
 from _variable cimport Variable as _Variable, CVariable
+from _variable cimport CommunicatorBackwardCallback
 from _variable import Context
 
 cimport _nd_array
@@ -395,6 +396,57 @@ cdef class Communicator:
             self.communicatorp.reduce_scatter(
                 cndarray_list, cndarray, division, group)
 
+    def all_reduce_callback(self, data, size_t pack_size, cpp_bool division=False, string group="world"):
+        """All reduce over data.
+
+        Note:
+            This function does not support shared parameters (such as RNNs) currently.
+
+        Args:
+            data (:obj:`NdArray` or list of :obj:`NdArray`)
+            pack_size (int): The number of values contained in the packed data.
+            division (bool): Flag to divide the reduce data by the
+                number of `contexts` added, or the number of devices.
+            group (string): Name of a group. This groups is used when the collective is called.
+
+        Example:
+
+        In case of the multi-process data parallel distributed training,
+
+        .. code-block:: python
+
+            # Communicator and Context
+            extension_module = "cuda.cudnn"
+            ctx = extension_context(extension_module)
+            comm = C.MultiProcessDataParalellCommunicator(ctx)
+            comm.init()
+
+            n_class = 2
+            b, c, h, w = 4, 1, 32, 32
+
+            # Data
+            x = nn.Variable([b, c, h, w])
+            y = nn.Variable([b, 1])
+
+            # Network setting
+            h = PF.convolution(x, 1, (3, 3), (1, 1), (1, 1))
+            pred = PF.affine(h, 2)
+            loss = F.mean(F.softmax_cross_entropy(pred, y))
+
+            loss.forward()
+            # AllReduce during backward
+            loss.backward(communicator_callbacks = comm.all_reduce_callback([v.grad for v in nn.get_parameters().values()], 1024 * 1024 * 2))
+
+        """
+        cdef vector[shared_ptr[CNdArray]] cndarray_list
+        if type(data) == list:
+            for x in data:
+                cndarray_list.push_back((< NdArray > x).arr)
+        else:
+            cndarray_list.push_back((< NdArray > data).arr)
+        return CommunicatorBackwardCallback.create_from_ccallback(
+            self.communicatorp.all_reduce_callback(cndarray_list, pack_size, division, group))
+
 
 def DataParalellCommunicator(CContext ctx):
     """Data Parallel Communicator for Distributed Training.

python/src/nnabla/parameter.py

@@ -183,7 +183,8 @@ class VariableInfo:
             if isinstance(initializer, numpy.ndarray):  # numpy init
                 param = nn.Variable(initializer.shape, need_grad=need_grad)
                 param.d = initializer
-            elif isinstance(initializer, nn.initializer.BaseInitializer):  # initializer init
+            # initializer init
+            elif isinstance(initializer, nn.initializer.BaseInitializer) or initializer.__name__ == "<lambda>":
                 assert shape is not None
                 param = nn.Variable(shape, need_grad=need_grad)
                 param.d = initializer(shape=param.shape)

python/src/nnabla/utils/data_source_implements.py

@@ -44,7 +44,7 @@ def _get_data(self, position):
         return self._load_func(self._order[position])
 
     def reset(self):
-        pass
+        super(SimpleDataSource, self).reset()
 
     def __init__(self, load_func, num_examples, shuffle=False, rng=None):
         super(SimpleDataSource, self).__init__(shuffle=shuffle, rng=rng)

python/src/nnabla/utils/load.py

@@ -460,6 +460,7 @@ class TrainingConfig:
     config.max_epoch = proto.training_config.max_epoch
     config.iter_per_epoch = proto.training_config.iter_per_epoch
     config.save_best = proto.training_config.save_best
+    config.monitor_interval = proto.training_config.monitor_interval if proto.training_config.monitor_interval > 0 else 10
     return config
 
 

python/src/nnabla/utils/network.py

@@ -168,7 +168,7 @@ def prepare_backward(self, backward_sequence, parameter_zero_grad=True):
         for v in backward_sequence.variables:
             v.need_grad = True
         for l in backward_sequence.loss_variables:
-            l.grad.fill(1.0 / l.shape[0])
+            l.grad.fill(1.0 / l.size)
 
     def backward(self, backward_sequence, parameter_zero_grad=True):
         self.prepare_backward(backward_sequence, parameter_zero_grad)