diff --git a/benchmark/paddle/image/vgg.py b/benchmark/paddle/image/vgg.py index b8429975f5c83..420884ed8e1ae 100644 --- a/benchmark/paddle/image/vgg.py +++ b/benchmark/paddle/image/vgg.py @@ -13,7 +13,7 @@ settings( batch_size=batch_size, - learning_rate=0.01 / batch_size, + learning_rate=0.001 / batch_size, learning_method=MomentumOptimizer(0.9), regularization=L2Regularization(0.0005 * batch_size)) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 8fdc382f0c1c4..b21fc43904d9a 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -1,17 +1,12 @@ # Find the CBlas and lapack libraries # -# It will search MKL, atlas, OpenBlas, reference-cblas in order. +# It will search MKLML, atlas, OpenBlas, reference-cblas in order. # # If any cblas implementation found, the following variable will be set. -# CBLAS_PROVIDER # one of MKL, ATLAS, OPENBLAS, REFERENCE +# CBLAS_PROVIDER # one of MKLML, ATLAS, OPENBLAS, REFERENCE # CBLAS_INC_DIR # the include directory for cblas. # CBLAS_LIBS # a list of libraries should be linked by paddle. # # Each library should be full path to object file. -# -# User should set one of MKL_ROOT, ATLAS_ROOT, OPENBLAS_ROOT, REFERENCE_CBLAS_ROOT -# during cmake. If none of them set, it will try to find cblas implementation in -# system paths. -# set(CBLAS_FOUND OFF) @@ -30,44 +25,6 @@ if(WITH_MKLML AND MKLML_INC_DIR AND MKLML_LIB) return() endif() -## Then find MKL. -set(INTEL_MKL_ROOT "/opt/intel/mkl" CACHE PATH "Folder contains intel mkl libs") -set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder contains env MKL") - -set(MKL_INCLUDE_SEARCH_PATHS - ${MKL_ROOT}/include - ${INTEL_MKL_ROOT}/include) -set(MKL_LIB_SEARCH_PATHS - ${MKL_ROOT}/lib - ${MKL_ROOT}/lib/intel64 - ${INTEL_MKL_ROOT}/lib - ${INTEL_MKL_ROOT}/lib/intel64) - -find_path(MKL_INC_DIR mkl.h PATHS - ${MKL_INCLUDE_SEARCH_PATHS}) -find_path(MKL_LAPACK_INC_DIR mkl_lapacke.h PATHS - ${MKL_INCLUDE_SEARCH_PATHS}) -find_library(MKL_CORE_LIB NAMES mkl_core PATHS - ${MKL_LIB_SEARCH_PATHS}) -find_library(MKL_SEQUENTIAL_LIB NAMES mkl_sequential PATHS - ${MKL_LIB_SEARCH_PATHS}) -find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS - ${MKL_LIB_SEARCH_PATHS}) - -if(MKL_LAPACK_INC_DIR AND MKL_INC_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64) - set(CBLAS_FOUND ON) - set(CBLAS_PROVIDER MKL) - set(CBLAS_INC_DIR ${MKL_INC_DIR} ${MKL_LAPACK_INC_DIR}) - set(CBLAS_LIBRARIES ${MKL_INTEL_LP64} ${MKL_SEQUENTIAL_LIB} ${MKL_CORE_LIB}) - - add_definitions(-DPADDLE_USE_MKL) - add_definitions(-DLAPACK_FOUND) - - message(STATUS "Found MKL (include: ${MKL_INC_DIR}, library: ${CBLAS_LIBRARIES})") - message(STATUS "Found lapack in MKL (include: ${MKL_LAPACK_INC_DIR})") - return() -endif() - ## Then find atlas. set(ATLAS_ROOT $ENV{ATLAS_ROOT} CACHE PATH "Folder contains Atlas") set(ATLAS_INCLUDE_SEARCH_PATHS diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 9686df0021900..5a06825beb73e 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -46,16 +46,20 @@ IF(${CBLAS_PROVIDER} STREQUAL "MKLML") MESSAGE(STATUS "Build MKLDNN with ${MKLDNN_MKLROOT}") ENDIF() +SET(MKLDNN_CFLAG "${CMAKE_C_FLAGS} -Wno-error=strict-overflow") +SET(MKLDNN_CXXFLAG "${CMAKE_CXX_FLAGS} -Wno-error=strict-overflow") ExternalProject_Add( ${MKLDNN_PROJECT} ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS ${MKLDNN_DEPENDS} GIT_REPOSITORY "https://github.com/01org/mkl-dnn.git" - GIT_TAG "v0.10" + GIT_TAG "v0.11" PREFIX ${MKLDNN_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL_DIR} CMAKE_ARGS -DMKLROOT=${MKLDNN_MKLROOT} + CMAKE_ARGS -DCMAKE_C_FLAGS=${MKLDNN_CFLAG} + CMAKE_ARGS -DCMAKE_CXX_FLAGS=${MKLDNN_CXXFLAG} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${MKLDNN_INSTALL_DIR} -DMKLROOT:PATH=${MKLDNN_MKLROOT} ) diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index 74f3279831357..20dbc32a738d9 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -27,8 +27,8 @@ ENDIF() INCLUDE(ExternalProject) SET(MKLML_PROJECT "extern_mklml") -SET(MKLML_VER "mklml_lnx_2018.0.20170720") -SET(MKLML_URL "https://github.com/01org/mkl-dnn/releases/download/v0.10/${MKLML_VER}.tgz") +SET(MKLML_VER "mklml_lnx_2018.0.1.20171007") +SET(MKLML_URL "https://github.com/01org/mkl-dnn/releases/download/v0.11/${MKLML_VER}.tgz") SET(MKLML_SOURCE_DIR "${THIRD_PARTY_PATH}/mklml") SET(MKLML_DOWNLOAD_DIR "${MKLML_SOURCE_DIR}/src/${MKLML_PROJECT}") SET(MKLML_DST_DIR "mklml") diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 3f86e456cfbe5..05d83ad58ef84 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -86,7 +86,7 @@ IF(NOT ${CBLAS_FOUND}) UPDATE_COMMAND "" CONFIGURE_COMMAND "" ) - + SET(CBLAS_PROVIDER openblas) IF(WITH_C_API) INSTALL(DIRECTORY ${CBLAS_INC_DIR} DESTINATION third_party/openblas) # Because libopenblas.a is a symbolic link of another library, thus need to @@ -115,7 +115,7 @@ INCLUDE_DIRECTORIES(${CBLAS_INC_DIR}) # linear algebra libraries for cc_library(xxx SRCS xxx.c DEPS cblas) SET(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/cblas_dummy.c) FILE(WRITE ${dummyfile} "const char * dummy = \"${dummyfile}\";") -IF(${CBLAS_PROVIDER} MATCHES MKL) +IF("${CBLAS_PROVIDER}" STREQUAL "MKLML") ADD_LIBRARY(cblas SHARED ${dummyfile}) ELSE() ADD_LIBRARY(cblas STATIC ${dummyfile}) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index c311783aa3187..b9c1dde97bc44 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -93,7 +93,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) if(NOT APPLE AND NOT ANDROID) find_package(Threads REQUIRED) link_libraries(${CMAKE_THREAD_LIBS_INIT}) - set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -ldl -lrt") + set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -pthread -ldl -lrt") endif(NOT APPLE AND NOT ANDROID) function(merge_static_libs TARGET_NAME) diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index d4e9d53e5c095..203506d7ab84e 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -82,6 +82,11 @@ maxout .. autoclass:: paddle.v2.layer.maxout :noindex: +roi_pool +-------- +.. autoclass:: paddle.v2.layer.roi_pool + :noindex: + Norm Layer ========== diff --git a/doc/api/v2/data.rst b/doc/api/v2/data.rst index fef87c4fbdb45..b56c7332cc284 100644 --- a/doc/api/v2/data.rst +++ b/doc/api/v2/data.rst @@ -2,112 +2,9 @@ Data Reader Interface and DataSets ================================== +.. toctree:: + :maxdepth: 1 -DataTypes -========= - -.. automodule:: paddle.v2.data_type - :members: - :noindex: - -DataFeeder -========== - -.. automodule:: paddle.v2.data_feeder - :members: - :noindex: - -Reader -====== - -.. automodule:: paddle.v2.reader - :members: - :noindex: - -.. automodule:: paddle.v2.reader.creator - :members: - :noindex: - -minibatch -========= - -.. automodule:: paddle.v2.minibatch - :members: - :noindex: - -Dataset -======= - -.. automodule:: paddle.v2.dataset - :members: - :noindex: - -mnist -+++++ - -.. automodule:: paddle.v2.dataset.mnist - :members: - :noindex: - -cifar -+++++ - -.. automodule:: paddle.v2.dataset.cifar - :members: - :noindex: - -conll05 -+++++++ - -.. automodule:: paddle.v2.dataset.conll05 - :members: get_dict,get_embedding,test - :noindex: - -imdb -++++ - -.. automodule:: paddle.v2.dataset.imdb - :members: - :noindex: - -imikolov -++++++++ - -.. automodule:: paddle.v2.dataset.imikolov - :members: - :noindex: - -movielens -+++++++++ - -.. automodule:: paddle.v2.dataset.movielens - :members: - :noindex: - -.. autoclass:: paddle.v2.dataset.movielens.MovieInfo - :noindex: - -.. autoclass:: paddle.v2.dataset.movielens.UserInfo - :noindex: - -sentiment -+++++++++ - -.. automodule:: paddle.v2.dataset.sentiment - :members: - :noindex: - -uci_housing -+++++++++++ - -.. automodule:: paddle.v2.dataset.uci_housing - :members: - :noindex: - -wmt14 -+++++ - -.. automodule:: paddle.v2.dataset.wmt14 - :members: - :noindex: - + data/data_reader.rst + data/image.rst + data/dataset.rst diff --git a/doc/api/v2/data/data_reader.rst b/doc/api/v2/data/data_reader.rst new file mode 100644 index 0000000000000..2ccfec9c28487 --- /dev/null +++ b/doc/api/v2/data/data_reader.rst @@ -0,0 +1,36 @@ +===================== +Data Reader Interface +===================== + + +DataTypes +========= + +.. automodule:: paddle.v2.data_type + :members: + :noindex: + +DataFeeder +========== + +.. automodule:: paddle.v2.data_feeder + :members: + :noindex: + +Reader +====== + +.. automodule:: paddle.v2.reader + :members: + :noindex: + +.. automodule:: paddle.v2.reader.creator + :members: + :noindex: + +minibatch +========= + +.. automodule:: paddle.v2.minibatch + :members: + :noindex: diff --git a/doc/api/v2/data/dataset.rst b/doc/api/v2/data/dataset.rst new file mode 100644 index 0000000000000..6a8ecc5bb1d85 --- /dev/null +++ b/doc/api/v2/data/dataset.rst @@ -0,0 +1,75 @@ +Dataset +======= + +.. automodule:: paddle.v2.dataset + :members: + :noindex: + +mnist ++++++ + +.. automodule:: paddle.v2.dataset.mnist + :members: + :noindex: + +cifar ++++++ + +.. automodule:: paddle.v2.dataset.cifar + :members: + :noindex: + +conll05 ++++++++ + +.. automodule:: paddle.v2.dataset.conll05 + :members: get_dict,get_embedding,test + :noindex: + +imdb +++++ + +.. automodule:: paddle.v2.dataset.imdb + :members: + :noindex: + +imikolov +++++++++ + +.. automodule:: paddle.v2.dataset.imikolov + :members: + :noindex: + +movielens ++++++++++ + +.. automodule:: paddle.v2.dataset.movielens + :members: + :noindex: + +.. autoclass:: paddle.v2.dataset.movielens.MovieInfo + :noindex: + +.. autoclass:: paddle.v2.dataset.movielens.UserInfo + :noindex: + +sentiment ++++++++++ + +.. automodule:: paddle.v2.dataset.sentiment + :members: + :noindex: + +uci_housing ++++++++++++ + +.. automodule:: paddle.v2.dataset.uci_housing + :members: + :noindex: + +wmt14 ++++++ + +.. automodule:: paddle.v2.dataset.wmt14 + :members: + :noindex: diff --git a/doc/api/v2/data/image.rst b/doc/api/v2/data/image.rst new file mode 100644 index 0000000000000..97651ffa6be56 --- /dev/null +++ b/doc/api/v2/data/image.rst @@ -0,0 +1,5 @@ +Image Interface +=============== + +.. automodule:: paddle.v2.image + :members: diff --git a/doc/design/mkldnn/README.MD b/doc/design/mkldnn/README.MD index fe8da907d9d45..16236763a7377 100644 --- a/doc/design/mkldnn/README.MD +++ b/doc/design/mkldnn/README.MD @@ -15,6 +15,7 @@ - [CMake](#cmake) - [Layers](#layers) - [Activations](#activations) + - [Weights](#weights) - [Unit Tests](#unit-tests) - [Protobuf Messages](#protobuf-messages) - [Python API](#python-api) @@ -45,17 +46,23 @@ Figure 1. PaddlePaddle on IA. ### Layers 所有MKL-DNN相关的C++ layers，都会按照PaddlePaddle的目录结构存放在 -`paddle/gserver/layers`中，并且文件名都会一以*Mkldnn*开头。 +`paddle/gserver/layers`中，并且文件名都会一以*MKLDNN*开头。 -所有MKL-DNN的layers都会继承于一个叫做`MkldnnLayer`的父类，该父类继承于PaddlePaddle的基类`Layer`。 +所有MKL-DNN的layers都会继承于一个叫做`MKLDNNLayer`的父类，该父类继承于PaddlePaddle的基类`Layer`。 + +在`MKLDNNLayer`中会提供一些必要的接口和函数，并且会写好`forward`和`backward`的基本逻辑。部分函数定义为纯虚函数，子类只需要实现这些函数即可。 ### Activations -由于在PaddlePaddle中，激活函数是独立于layer概念的，所以会在`paddle/gserver/activations`目录下添加一个`MkldnnActivation.h`文件定义一些用于MKL-DNN的接口，实现方法还是会在`ActivationFunction.cpp`文件。 +由于在PaddlePaddle中，激活函数是独立于layer概念的，所以会在`paddle/gserver/activations`目录下添加`MKLDNNActivation.h`和`MKLDNNActivation.cpp`文件用于定义和使用MKL-DNN的接口。 -### Unit Tests -会在`paddle/gserver/test`目录下添加`test_Mkldnn.cpp`和`MkldnnTester.*`用于MKL-DNN的测试。 +### Weights +由于有些layer是含有参数的，我们会尽量让MKL-DNN的参数与PaddlePaddle中`parameter`共享一块内存。 +同时，由于MKL-DNN在训练时使用的参数layout可能与PaddlePaddle默认的`nchw`不一致，我们会在网络训练的开始和结束时分别转换这个layout，使得最终保存的参数格式与PaddlePaddle一致。 -Activation的测试，计划在PaddlePaddle原有的测试文件上直接添加新的测试type。 +### Unit Tests +会在`paddle/gserver/test`目录下添加`test_MKLDNN.cpp`和`MKLDNNTester.*`用于MKL-DNN的测试。 +测试分为每个layer(或activation)的单元测试和简单网络的整体测试。 +每个测试会对比PaddlePaddle中CPU算出的结果与MKL-DNN的结果，小于某个比较小的阈值认为通过。 ### Protobuf Messages 根据具体layer的需求可能会在`proto/ModelConfig.proto`里面添加必要的选项。 @@ -82,7 +89,7 @@ if use_mkldnn 会在`v1_api_demo`目录下添加一个`mkldnn`的文件夹，里面放入一些用于MKL-DNN测试的demo脚本。 ### Benchmarking -会考虑添加部分逻辑在`benchmark/paddle/image/run.sh`，添加使用MKL-DNN的测试。 +会添加`benchmark/paddle/image/run_mkldnn.sh`，用于测试使用MKL-DNN之后的性能。 ### Others 1. 如果在使用MKL-DNN的情况下，会把CPU的Buffer对齐为64。 @@ -94,14 +101,16 @@ if use_mkldnn 我们总结出一些特别需要注意的点： -1. 使用**deviceId_**。为了尽可能少的在父类Layer中添加变量或者函数，我们决定使用已有的`deviceId_`变量来区分layer的属性，定义`-2`为`MkldnnLayer`特有的设备ID。 +1. 使用**deviceId_**。为了尽可能少的在父类Layer中添加变量或者函数，我们决定使用已有的`deviceId_`变量来区分layer的属性，定义`-2`为`MKLDNNLayer`特有的设备ID。 2. 重写父类Layer的**init**函数，修改`deviceId_`为`-2`，代表这个layer是用于跑在MKL-DNN的环境下。 -3. 创建`MkldnnMatrix`，用于管理MKL-DNN会用到的相关memory函数、接口以及会用的到格式信息。 -4. 创建`MkldnnBase`，定义一些除了layer和memory相关的类和函数。包括MKL-DNN会用到`MkldnnStream`和`CpuEngine`，和未来可能还会用到`FPGAEngine`等。 -5. 在**Argument**里添加两个`MkldnnMatrixPtr`，取名为`mkldnnValue`和`mkldnnGrad`，用于存放`MkldnnLayer`会用到的memory buffer。 并且添加函数cvt(会修改为一个更加合适的函数名)，用于处理"CPU device"和"MKL-DNN device"之间memory的相互转化。 -6. 在父类`Layer`中的`getOutput`函数中添加一段逻辑，用于判断`deviceId`，并针对device在MKL-DNN和CPU之间不统一的情况，做一个前期转换。 也就是调用`Argument`的cvt函数把output统一到需要的device上。 -7. 在原来的`FLAGS`中添加一个`use_mkldnn`的flag，用于选择是否使用MKL-DNN的相关功能。 -8. 关于MKLDNN参数的保存。由于MKLDNN参数的格式与PaddlePaddle原有的格式存在不一样的情况，所以需要在保存参数时同时保存该格式信息。目前准备扩展[Header](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/parameter/Parameter.h#L247)里面的`int32_t version`。这个值不管是在v1还是在v2里面，一直保存的是0，所以可以充分利用这个信息，定义一个枚举处理所有MKLDNN的参数格式，从而`MKLDNNLayer`就可以从输入的参数中获取需要的格式信息。 +3. 创建`MKLDNNMatrix`，同时继承`CpuMatrix`和`mkldnn::memory`。用于管理MKL-DNN会用到的相关memory函数、接口以及会用的到格式信息。 +4. 创建`MKLDNNBase`，定义一些除了layer和memory相关的类和函数。包括MKL-DNN会用到`MKLDNNStream`和`CPUEngine`，和未来可能还会用到`FPGAEngine`等。 +5. 每个`MKLDNNlayer`都会有`inVal_`,`inGrad_`,`outVal_`和`outGrad_`，分别代表input value， input gradient，output value和output gradient。他们会存放MKL-DNN用到的internal memory。同时还会定义以*ext*开头的`MKLDNNMatrix`(表示external的memory)，主要是在格式与PaddlePaddle默认的`nchw`格式不匹配时，用于转换内存的工作。必要的转换函数也会在`MKLDNNLayer`中提前定义好，每个子类只需要调用定义好的reset buffer函数即可。 +6. 每个`MKLDNNlayer`的resetbuffer相关的函数（包括reset input、output的Value和grad），他们会根据输入参数reset internal和external的memory，当然这两者也可以相等，即表示不需要转换。只需要把握一个原则，每个`MKLDNNlayer`的子类，只需要使用internal的memory就可以了，所有external的转换工作在父类的reset函数中都提前准备好了。 +7. 一般来说，external的memory会尽量与PaddlePaddle中的`value`和`grad`共享内存。同时每个`MKLDNNLayer`中的external output value和gradient(也就是`extOutVal_`和`extOutGrad_`)必须分别与`output_.value`和`output_.grad`共享内存，因为PaddlePaddle的activation会直接使用`output_.value`和`output_.grad`。如果不需要external的buffer用于转换，那么internal的buffer也会与他们共享内存。 +8. 如果MKL-DNN layer的后面接有cpu device，那么就会使`output_.value`与`extOutVal_`共享内存，同时数据格式就是`nchw`，这样下一个cpu device就能拿到正确的数据。在有cpu device的时候，external的memory的格式始终是`nchw`或者`nc`。 +9. 由于MKL-DNN的输出操作都是覆盖data的，不是在原来的数据上累加，所以当网络出现分支时，在`backward`时会需要merge不同layer的梯度。`MKLDNNlayer`中会实现merge的方法，此时每个小分支的input gradient会先临时保存在一个`MKLDNNMatrix`中，由分支处的layer负责求和，并把结果放到这个layer的`output_.grad`中。所以整体上，每个子类并不会需要关心分支的事情，也是在父类都实现好了。 +10. 在原来的`FLAGS`中添加一个`use_mkldnn`的flag，用于选择是否使用MKL-DNN的相关功能。 ## References diff --git a/doc/design/ops/images/LOD-and-shape-changes-during-decoding.jpg b/doc/design/ops/images/LOD-and-shape-changes-during-decoding.jpg new file mode 100644 index 0000000000000..8b0d90f7b9d81 Binary files /dev/null and b/doc/design/ops/images/LOD-and-shape-changes-during-decoding.jpg differ diff --git a/doc/design/ops/sequence_decoder.md b/doc/design/ops/sequence_decoder.md new file mode 100644 index 0000000000000..9007aae7a8355 --- /dev/null +++ b/doc/design/ops/sequence_decoder.md @@ -0,0 +1,245 @@ +# Design: Sequence Decoder Generating LoDTensors +In tasks such as machine translation and image to text, +a [sequence decoder](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.md) is necessary to generate sequences. + +This documentation describes how to implement the sequence decoder as an operator. + +## Beam Search based Decoder +The [beam search algorithm](https://en.wikipedia.org/wiki/Beam_search) is necessary when generating sequences, +it is a heuristic search algorithm that explores the paths by expanding the most promising node in a limited set. + +In the old version of PaddlePaddle, a C++ class `RecurrentGradientMachine` implements the general sequence decoder based on beam search, +due to the complexity, the implementation relays on a lot of special data structures, +quite trivial and hard to be customized by users. + +There are a lot of heuristic tricks in the sequence generation tasks, +so the flexibility of sequence decoder is very important to users. + +During PaddlePaddle's refactoring work, +some new concept is proposed such as [LoDTensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/lod_tensor.md) and [TensorArray](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/tensor_array.md) that can better support sequence usage, +and they can help to make the implementation of beam search based sequence decoder **more transparent and modular** . + +For example, the RNN sates, candidates IDs and probabilities of beam search can be represented as `LoDTensors`; +the selected candidate's IDs in each time step can be stored in a `TensorArray`, and `Packed` to the sentences translated. + +## Changing LoD's absolute offset to relative offsets +The current `LoDTensor` is designed to store levels of variable-length sequences, +it stores several arrays of integers each represents a level. + +The integers in each level represents the begin and end (not inclusive) offset of a sequence **in the underlying tensor**, +let's call this format the **absolute-offset LoD** for clear. + +The relative-offset LoD can fast retrieve any sequence but fails to represent empty sequences, for example, a two-level LoD is as follows +```python +[[0, 3, 9] + [0, 2, 3, 3, 3, 9]] +``` +The first level tells that there are two sequences: +- the first's offset is `[0, 3)` +- the second's offset is `[3, 9)` + +while on the second level, there are several empty sequences that both begin and end at `3`. +It is impossible to tell how many empty second-level sequences exist in the first-level sequences. + +There are many scenarios that relay on empty sequence representation, +such as machine translation or image to text, one instance has no translations or the empty candidate set for a prefix. + +So let's introduce another format of LoD, +it stores **the offsets of the lower level sequences** and is called **relative-offset** LoD. + +For example, to represent the same sequences of the above data + +```python +[[0, 3, 6] + [0, 2, 3, 3, 3, 9]] +``` + +the first level represents that there are two sequences, +their offsets in the second-level LoD is `[0, 3)` and `[3, 5)`. + +The second level is the same with the relative offset example because the lower level is a tensor. +It is easy to find out the second sequence in the first-level LoD has two empty sequences. + +The following demos are based on relative-offset LoD. + +## Usage in a simple machine translation model +Let's start from a simple machine translation model that is simplified from [machine translation chapter](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation) to draw a simple blueprint of what a sequence decoder can do and how to use it. + +The model has an encoder that learns the semantic vector from a sequence, +and a decoder which uses the sequence decoder to generate new sentences. + +**Encoder** +```python +import paddle as pd + +dict_size = 8000 +source_dict_size = dict_size +target_dict_size = dict_size +word_vector_dim = 128 +encoder_dim = 128 +decoder_dim = 128 +beam_size = 5 +max_length = 120 + +# encoder +src_word_id = pd.data( + name='source_language_word', + type=pd.data.integer_value_sequence(source_dict_dim)) +src_embedding = pd.embedding(size=source_dict_size, size=word_vector_dim) + +src_word_vec = pd.lookup(src_embedding, src_word_id) + +encoder_out_seq = pd.gru(input=src_word_vec, size=encoder_dim) + +encoder_ctx = pd.last_seq(encoder_out_seq) +# encoder_ctx_proj is the learned semantic vector +encoder_ctx_proj = pd.fc( + encoder_ctx, size=decoder_dim, act=pd.activation.Tanh(), bias=None) +``` + +**Decoder** + +```python +def generate(): + decoder = pd.while_loop() + with decoder.step(): + decoder_mem = decoder.memory(init=encoder_ctx) # mark the memory + generated_ids = decoder.memory() # TODO init to batch_size s + generated_scores = decoder.memory() # TODO init to batch_size 1s or 0s + + target_word = pd.lookup(trg_embedding, gendrated_ids) + # expand encoder_ctx's batch to fit target_word's lod + # for example + # decoder_mem.lod is + # [[0 1 3], + # [0 1 3 6]] + # its tensor content is [a1 a2 a3 a4 a5] + # which means there are 2 sentences to translate + # - the first sentence has 1 translation prefixes, the offsets are [0, 1) + # - the second sentence has 2 translation prefixes, the offsets are [1, 3) and [3, 6) + # the target_word.lod is + # [[0, 1, 6] + # [0, 2, 4, 7, 9 12]] + # which means 2 sentences to translate, each has 1 and 5 prefixes + # the first prefix has 2 candidates + # the following has 2, 3, 2, 3 candidates + # the encoder_ctx_expanded's content will be + # [a1 a1 a2 a2 a3 a3 a3 a4 a4 a5 a5 a5] + encoder_ctx_expanded = pd.lod_expand(encoder_ctx, target_word) + decoder_input = pd.fc( + act=pd.activation.Linear(), + input=[target_word, encoder_ctx], + size=3 * decoder_dim) + gru_out, cur_mem = pd.gru_step( + decoder_input, mem=decoder_mem, size=decoder_dim) + scores = pd.fc( + gru_out, + size=trg_dic_size, + bias=None, + act=pd.activation.Softmax()) + # K is an config + topk_scores, topk_ids = pd.top_k(scores, K) + topk_generated_scores = pd.add_scalar(topk_scores, generated_scores) + + selected_ids, selected_generation_scores = decoder.beam_search( + topk_ids, topk_generated_scores) + + # update the states + decoder_mem.update(cur_mem) # tells how to update state + generated_ids.update(selected_ids) + generated_scores.update(selected_generation_scores) + + decoder.output(selected_ids) + decoder.output(selected_generation_scores) + +translation_ids, translation_scores = decoder() +``` +The `decoder.beam_search` is a operator that given the candidates and the scores of translations including the candidates, +return the result of the beam search algorithm. + +In this way, users can customize anything on the inputs or outputs of beam search, for example, two ways to prune some translation prefixes + +1. meke the correspondind elements in `topk_generated_scores` zero or some small values, beam_search will discard this candidate. +2. remove some specific candidate in `selected_ids` +3. get the final `translation_ids`, remove the translation sequence in it. + +The implementation of sequence decoder can reuse the C++ class [RNNAlgorithm](https://github.com/Superjom/Paddle/blob/68cac3c0f8451fe62a4cdf156747d6dc0ee000b3/paddle/operators/dynamic_recurrent_op.h#L30), +so the python syntax is quite similar to a [RNN](https://github.com/Superjom/Paddle/blob/68cac3c0f8451fe62a4cdf156747d6dc0ee000b3/doc/design/block.md#blocks-with-for-and-rnnop). + +Both of them are two-level `LoDTensors` + +- the first level represents `batch_size` of (source) sentences; +- the second level represents the candidate ID sets for translation prefix. + +for example, 3 source sentences to translate, and has 2, 3, 1 candidates. + +Unlike an RNN, in sequence decoder, the previous state and the current state have different LoD and shape, +a `lod_expand` operator is used to expand the LoD of the previous state to fit the current state. + +For example, the previous state + +* LoD is `[0, 1, 3][0, 2, 5, 6]` +* content of tensor is `a1 a2 b1 b2 b3 c1` + +the current state stored in `encoder_ctx_expanded` + +* LoD is `[0, 2, 7][0 3 5 8 9 11 11]` +* the content is + - a1 a1 a1 (a1 has 3 candidates, so the state should be copied 3 times for each candidates) + - a2 a2 + - b1 b1 b1 + - b2 + - b3 b3 + - None (c1 has 0 candidates, so c1 is dropped) + +Benefit from the relative offset LoD, empty candidate set can be represented naturally. + +the status in each time step can be stored in `TensorArray`, and `Pack`ed to a final LoDTensor, the corresponding syntax is + +```python +decoder.output(selected_ids) +decoder.output(selected_generation_scores) +``` + +the `selected_ids` is the candidate ids for the prefixes, +it will be `Packed` by `TensorArray` to a two-level `LoDTensor`, +the first level represents the source sequences, +the second level represents generated sequences. + +Pack the `selected_scores` will get a `LoDTensor` that stores scores of each candidate of translations. + +Pack the `selected_generation_scores` will get a `LoDTensor`, and each tail is the probability of the translation. + +## LoD and shape changes during decoding +

+ +

+ +According the image above, the only phrase to change LoD is beam search. + +## Beam search design +The beam search algorthm will be implemented as one method of the sequence decoder, it has 3 inputs + +1. `topk_ids`, top K candidate ids for each prefix. +2. `topk_scores`, the corresponding scores for `topk_ids` +3. `generated_scores`, the score of the prefixes. + +All of the are LoDTensors, so that the sequence affilication is clear. +Beam search will keep a beam for each prefix and select a smaller candidate set for each prefix. + +It will return three variables + +1. `selected_ids`, the final candidate beam search function selected for the next step. +2. `selected_scores`, the scores for the candidates. +3. `generated_scores`, the updated scores for each prefixes (with the new candidates appended). + +## Introducing the LoD-based `Pack` and `Unpack` methods in `TensorArray` +The `selected_ids`, `selected_scores` and `generated_scores` are LoDTensors, +and they exist in each time step, +so it is natural to store them in arrays. + +Currently, PaddlePaddle has a module called `TensorArray` which can store an array of tensors, +the results of beam search are better to store in a `TensorArray`. + +The `Pack` and `UnPack` in `TensorArray` are used to package tensors in the array to a `LoDTensor` or split the `LoDTensor` to an array of tensors. +It needs some extensions to support pack or unpack an array of `LoDTensors`. diff --git a/doc/faq/local/index_cn.rst b/doc/faq/local/index_cn.rst index 0e939a2671ace..b331d9d36e6a2 100644 --- a/doc/faq/local/index_cn.rst +++ b/doc/faq/local/index_cn.rst @@ -99,7 +99,7 @@ PaddlePaddle支持Sparse的训练，sparse训练需要训练特征是 :code:`spa 利用更多的计算资源 ++++++++++++++++++ -利用更多的计算资源可以分为一下几个方式来进行\: +利用更多的计算资源可以分为以下几个方式来进行\: * 单机CPU训练 diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index c823d7e9fcd63..6cfc9536f20e8 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -214,7 +214,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, ```cpp // if use Eigen unsupported module before include head files - #define EIGEN_USE_GPU + // #define EIGEN_USE_GPU namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL(mul, ops::MulKernel); diff --git a/paddle/capi/Matrix.cpp b/paddle/capi/Matrix.cpp index 4547afaf1dc9a..53a36f8f20d11 100644 --- a/paddle/capi/Matrix.cpp +++ b/paddle/capi/Matrix.cpp @@ -54,6 +54,46 @@ paddle_error paddle_matrix_set_row(paddle_matrix mat, return kPD_NO_ERROR; } +PD_API paddle_error paddle_matrix_set_value(paddle_matrix mat, + paddle_real* value) { + if (mat == nullptr || value == nullptr) return kPD_NULLPTR; + auto ptr = cast(mat); + if (ptr->mat == nullptr) return kPD_NULLPTR; + paddle::real* buf = ptr->mat->getRowBuf(0); + size_t width = ptr->mat->getWidth(); + size_t height = ptr->mat->getHeight(); + if (ptr->mat->useGpu()) { +#ifdef PADDLE_WITH_CUDA + hl_memcpy(buf, value, sizeof(paddle::real) * width * height); +#else + return kPD_NOT_SUPPORTED; +#endif + } else { + std::copy(value, value + width * height, buf); + } + return kPD_NO_ERROR; +} + +PD_API paddle_error paddle_matrix_get_value(paddle_matrix mat, + paddle_real* result) { + if (mat == nullptr || result == nullptr) return kPD_NULLPTR; + auto ptr = cast(mat); + if (ptr->mat == nullptr) return kPD_NULLPTR; + paddle::real* buf = ptr->mat->getRowBuf(0); + size_t width = ptr->mat->getWidth(); + size_t height = ptr->mat->getHeight(); + if (ptr->mat->useGpu()) { +#ifdef PADDLE_WITH_CUDA + hl_memcpy(result, buf, width * height * sizeof(paddle::real)); +#else + return kPD_NOT_SUPPORTED; +#endif + } else { + std::copy(buf, buf + width * height, result); + } + return kPD_NO_ERROR; +} + paddle_error paddle_matrix_get_row(paddle_matrix mat, uint64_t rowID, paddle_real** rawRowBuffer) { diff --git a/paddle/capi/examples/model_inference/dense/main.c b/paddle/capi/examples/model_inference/dense/main.c index 3e6bd5285058a..876af2aa7615c 100644 --- a/paddle/capi/examples/model_inference/dense/main.c +++ b/paddle/capi/examples/model_inference/dense/main.c @@ -27,18 +27,20 @@ int main() { CHECK(paddle_arguments_resize(in_args, 1)); // Create input matrix. - paddle_matrix mat = paddle_matrix_create(/* sample_num */ 1, + paddle_matrix mat = paddle_matrix_create(/* sample_num */ 10, /* size */ 784, /* useGPU */ false); srand(time(0)); - paddle_real* array; - // Get First row. - CHECK(paddle_matrix_get_row(mat, 0, &array)); + std::vector input; + input.resize(784 * 10); - for (int i = 0; i < 784; ++i) { - array[i] = rand() / ((float)RAND_MAX); + for (int i = 0; i < input.size(); ++i) { + input[i] = rand() / ((float)RAND_MAX); } + + // Set value for the input matrix + CHECK(paddle_matrix_set_value(mat, input.data())); CHECK(paddle_arguments_set_value(in_args, 0, mat)); @@ -51,11 +53,17 @@ int main() { CHECK(paddle_arguments_get_value(out_args, 0, prob)); - CHECK(paddle_matrix_get_row(prob, 0, &array)); + std::std::vector result; + int height; + int width; + + CHECK(paddle_matrix_get_shape(prob, &height, &width); + result.resize(height * width); + CHECK(paddle_matrix_get_value(prob, result.data())); printf("Prob: "); - for (int i = 0; i < 10; ++i) { - printf("%.2f ", array[i]); + for (int i = 0; i < height * width; ++i) { + printf("%.2f ", result[i]); } printf("\n"); diff --git a/paddle/capi/matrix.h b/paddle/capi/matrix.h index f15f7f3bbbd14..bb5223f8a275f 100644 --- a/paddle/capi/matrix.h +++ b/paddle/capi/matrix.h @@ -70,6 +70,16 @@ PD_API paddle_error paddle_matrix_set_row(paddle_matrix mat, uint64_t rowID, paddle_real* rowArray); +/** + * @brief paddle_matrix_set_value Set value to matrix. + * @param mat Target Matrix + * @param value Row data. + * @return paddle_error + * @note value should contain enough element of data to init the mat + */ +PD_API paddle_error paddle_matrix_set_value(paddle_matrix mat, + paddle_real* value); + /** * @brief PDMatGetRow Get raw row buffer from matrix * @param [in] mat Target matrix @@ -81,6 +91,15 @@ PD_API paddle_error paddle_matrix_get_row(paddle_matrix mat, uint64_t rowID, paddle_real** rawRowBuffer); +/** + * @brief copy data from the matrix + * @param [in] mat Target matrix + * @param [out] result pointer to store the matrix data + * @return paddle_error + * @note the space of the result should allocated before invoke this API + */ +PD_API paddle_error paddle_matrix_get_value(paddle_matrix mat, + paddle_real* result); /** * @brief PDMatCreateNone Create None Matrix * @return diff --git a/paddle/capi/tests/test_Matrix.cpp b/paddle/capi/tests/test_Matrix.cpp index 4bf9a9d6a9f91..6940c28448a89 100644 --- a/paddle/capi/tests/test_Matrix.cpp +++ b/paddle/capi/tests/test_Matrix.cpp @@ -45,3 +45,49 @@ TEST(CAPIMatrix, createNone) { paddle_matrix mat = paddle_matrix_create_none(); ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_destroy(mat)); } + +TEST(CAPIMatrix, cpu_get_set_value) { + paddle_matrix mat = paddle_matrix_create(128, 32, false); + std::vector sample; + std::vector result; + sample.resize(128 * 32); + result.resize(128 * 32); + for (size_t i = 0; i < sample.size(); ++i) { + sample[i] = 1.0 / (i + 1.0); + } + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_set_value(mat, sample.data())); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_value(mat, result.data())); + for (size_t i = 0; i < sample.size(); ++i) { + ASSERT_NEAR(sample[i], result[i], 1e-5); + } + + uint64_t height, width; + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_shape(mat, &height, &width)); + ASSERT_EQ(128UL, height); + ASSERT_EQ(32UL, width); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_destroy(mat)); +} + +#ifdef PADDLE_WITH_CUDA +TEST(CAPIMatrix, gpu_get_set_value) { + paddle_matrix mat = paddle_matrix_create(128, 32, true); + std::vector sample; + std::vector result; + sample.resize(128 * 32); + result.resize(128 * 32); + for (size_t i = 0; i < sample.size(); ++i) { + sample[i] = 1.0 / (i + 1.0); + } + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_set_value(mat, sample.data())); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_value(mat, result.data())); + for (size_t i = 0; i < sample.size(); ++i) { + ASSERT_NEAR(sample[i], result[i], 1e-5); + } + + uint64_t height, width; + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_shape(mat, &height, &width)); + ASSERT_EQ(128UL, height); + ASSERT_EQ(32UL, width); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_destroy(mat)); +} +#endif diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index ed94540c268e5..b3b9c45ded95c 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -321,8 +321,6 @@ static void CreateGradVarInBlock( auto* param = block_desc->FindVarRecursive(pname); auto* grad = block_desc->FindVar(arg); if (param == nullptr) { - LOG(WARNING) << "Cannot find forward variable of " << arg - << ". Set its gradient to FP32"; grad->SetDataType(DataType::FP32); } else { grad->SetDataType(param->GetDataType()); @@ -379,6 +377,12 @@ std::vector> MakeOpGrad( return grad_op_descs; } +static BlockDescBind* CreateStepBlock( + ProgramDescBind& program_desc, + std::unordered_set* no_grad_vars, + std::unordered_map* grad_to_var, + int step_block_idx); + std::vector> MakeBlockBackward( ProgramDescBind& program_desc, int block_idx, std::unordered_set* no_grad_vars, @@ -394,13 +398,13 @@ std::vector> MakeBlockBackward( if ((*it)->Type() == "recurrent") { int step_block_idx = (*it)->GetBlockAttr("step_block"); - auto backward_block_op_descs = MakeBlockBackward( - program_desc, step_block_idx, no_grad_vars, grad_to_var); + BlockDescBind* backward_block = CreateStepBlock( + program_desc, no_grad_vars, grad_to_var, step_block_idx); + op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); + } else if ((*it)->Type() == "conditional_block") { BlockDescBind* backward_block = - program_desc.AppendBlock(*program_desc.MutableBlock(step_block_idx)); - for (auto& ptr : backward_block_op_descs) { - backward_block->AppendAllocatedOp(std::move(ptr)); - } + CreateStepBlock(program_desc, no_grad_vars, grad_to_var, + (*it)->GetBlockAttr("block")); op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); } else { op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var); @@ -408,6 +412,11 @@ std::vector> MakeBlockBackward( for (const auto& desc : op_grads) { for (const std::string& out_name : desc->OutputArgumentNames()) { + if (out_name.find("@GRAD") == std::string::npos) { + // Not all outputs of a backward operator is a gradient. Only gradient + // need to be sum. Skip variables are not gradient. + continue; + } dup_out_ops[out_name].emplace_back(grad_desc_idx); } ++grad_desc_idx; @@ -446,6 +455,21 @@ std::vector> MakeBlockBackward( return backward_descs; } +static BlockDescBind* CreateStepBlock( + ProgramDescBind& program_desc, + std::unordered_set* no_grad_vars, + std::unordered_map* grad_to_var, + int step_block_idx) { + auto backward_block_op_descs = MakeBlockBackward(program_desc, step_block_idx, + no_grad_vars, grad_to_var); + BlockDescBind* backward_block = + program_desc.AppendBlock(*program_desc.MutableBlock(step_block_idx)); + for (auto& ptr : backward_block_op_descs) { + backward_block->AppendAllocatedOp(move(ptr)); + } + return backward_block; +} + ParamGradInfoMap AppendBackward( ProgramDescBind& program_desc, const VarDescBind& target, const std::unordered_set& no_grad_vars) { diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 4e8d630c26346..d485cdf610927 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -21,7 +21,7 @@ #include "paddle/framework/var_desc.h" #include "paddle/operators/net_op.h" -USE_OP(fill_constant); +USE_NO_KERNEL_OP(fill_constant); namespace paddle { namespace framework { diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index 9e3d597f3a2c8..11764810e1d40 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -50,6 +50,15 @@ VarDescBind *BlockDescBind::FindVarRecursive(const std::string &name) const { return it->second.get(); } +VarDescBind *BlockDescBind::FindRecursiveOrCreateVar( + const std::string &name_bytes) { + VarDescBind *res = FindVarRecursive(name_bytes); + if (res == nullptr) { + res = Var(name_bytes); + } + return res; +} + bool BlockDescBind::HasVarRecursive(const std::string &name) const { return FindVarRecursive(name) != nullptr; } diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 26adf6a20ff09..8e967e5378eb4 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -58,6 +58,8 @@ class BlockDescBind { VarDescBind *FindVarRecursive(const std::string &name_bytes) const; + VarDescBind *FindRecursiveOrCreateVar(const std::string &name_bytes); + bool HasVarRecursive(const std::string &var_name) const; std::set LocalVarNames() const { diff --git a/paddle/framework/data_type.h b/paddle/framework/data_type.h index c5ae7b185460c..3ec88d7a72c33 100644 --- a/paddle/framework/data_type.h +++ b/paddle/framework/data_type.h @@ -34,6 +34,21 @@ inline DataType ToDataType(std::type_index type) { } } +inline std::type_index ToTypeIndex(DataType type) { + switch (type) { + case DataType::FP32: + return typeid(float); + case DataType::FP64: + return typeid(double); + case DataType::INT32: + return typeid(int); + case DataType::INT64: + return typeid(int64_t); + default: + PADDLE_THROW("Not support type %d", type); + } +} + template inline void VisitDataType(DataType type, Visitor visitor) { switch (type) { diff --git a/paddle/framework/ddim.cc b/paddle/framework/ddim.cc index 10c785e04c4fa..53b899a23997b 100644 --- a/paddle/framework/ddim.cc +++ b/paddle/framework/ddim.cc @@ -79,6 +79,13 @@ DDim make_ddim(const std::vector& dims) { return result; } +DDim make_ddim(const std::vector& dims) { + std::vector res(dims.size()); + std::transform(dims.begin(), dims.end(), res.begin(), + [](int d) { return static_cast(d); }); + return make_ddim(res); +} + /// @cond HIDDEN // XXX For some reason, putting this in an anonymous namespace causes errors class DynamicMutableIndexer : public boost::static_visitor { diff --git a/paddle/framework/ddim.h b/paddle/framework/ddim.h index aa773868ab4b6..4ca5e49566b7e 100644 --- a/paddle/framework/ddim.h +++ b/paddle/framework/ddim.h @@ -81,6 +81,8 @@ struct DDim { */ DDim make_ddim(const std::vector& dims); +DDim make_ddim(const std::vector& dims); + /** * \brief Make a DDim from an initializer list * diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index e7cba9e702ce0..39c8def82e1eb 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -357,7 +357,8 @@ void OpDescBind::InferVarType(BlockDescBind *block) const { "LOD_TENSOR"; for (auto &out_pair : this->outputs_) { for (auto &out_var_name : out_pair.second) { - block->Var(out_var_name)->SetType(VarDesc::LOD_TENSOR); + block->FindRecursiveOrCreateVar(out_var_name) + ->SetType(VarDesc::LOD_TENSOR); } } } diff --git a/paddle/framework/scope.cc b/paddle/framework/scope.cc index fb2c69105627f..9428b8a07ea0a 100644 --- a/paddle/framework/scope.cc +++ b/paddle/framework/scope.cc @@ -98,5 +98,23 @@ void Scope::DeleteScope(Scope* scope) { delete scope; } +void Scope::Rename(const std::string& origin_name, + const std::string& new_name) const { + auto origin_it = vars_.find(origin_name); + PADDLE_ENFORCE(origin_it != vars_.end(), + "Cannot find original variable with name %s", origin_name); + auto new_it = vars_.find(new_name); + PADDLE_ENFORCE(new_it == vars_.end(), + "The variable with name %s is already in the scope", new_name); + vars_[new_name] = origin_it->second; + vars_.erase(origin_it); +} + +std::string Scope::Rename(const std::string& origin_name) const { + auto var_name = string::Sprintf("%p.%d", this, vars_.size()); + Rename(origin_name, var_name); + return var_name; +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/scope.h b/paddle/framework/scope.h index fb66094939414..c2aafb6ad825f 100644 --- a/paddle/framework/scope.h +++ b/paddle/framework/scope.h @@ -68,11 +68,18 @@ class Scope { // enumerate all the variables current contains. std::vector GetAllNames(bool recursive = false) const; + // Rename variable to a new name + void Rename(const std::string& origin_name, + const std::string& new_name) const; + + // Rename variable to a new name and return the new name + std::string Rename(const std::string& origin_name) const; + private: // Call Scope::NewScope for a sub-scope. explicit Scope(Scope const* parent) : parent_(parent) {} - std::unordered_map vars_; + mutable std::unordered_map vars_; mutable std::list kids_; Scope const* parent_{nullptr}; diff --git a/paddle/framework/var_type.h b/paddle/framework/var_type.h index d060196bb2c47..0f19870bec3e6 100644 --- a/paddle/framework/var_type.h +++ b/paddle/framework/var_type.h @@ -27,10 +27,32 @@ inline VarDesc::VarType ToVarType(std::type_index type) { return VarDesc_VarType_LOD_RANK_TABLE; } else if (type.hash_code() == typeid(LoDTensorArray).hash_code()) { return VarDesc_VarType_LOD_TENSOR_ARRAY; + } else if (type.hash_code() == typeid(SelectedRows).hash_code()) { + return VarDesc_VarType_SELECTED_ROWS; } else { PADDLE_THROW("ToVarType:Unsupported type %s", type.name()); } } +template +inline void VisitVarType(const Variable& var, Visitor visitor) { + switch (ToVarType(var.Type())) { + case VarDesc_VarType_LOD_TENSOR: + visitor(var.Get()); + return; + case VarDesc_VarType_LOD_RANK_TABLE: + visitor(var.Get()); + return; + case VarDesc_VarType_LOD_TENSOR_ARRAY: + visitor(var.Get()); + return; + case VarDesc_VarType_SELECTED_ROWS: + visitor(var.Get()); + return; + default: + PADDLE_THROW("Not supported visit type, %d", ToVarType(var.Type())); + } +} + } // namespace framework } // namespace paddle diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index 4fd72d64a90ae..9b2779b42cad3 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -45,6 +45,7 @@ if(WITH_GPU) add_simple_unittest(BlockExpandOpTest) add_simple_unittest(CropOpTest) add_simple_unittest(SwitchOpTest) + add_simple_unittest(ScaleSubRegionOpTest) endif() add_simple_unittest(Im2ColTest) diff --git a/paddle/function/FunctionTest.h b/paddle/function/FunctionTest.h index ba446bf92da26..370940532ef40 100644 --- a/paddle/function/FunctionTest.h +++ b/paddle/function/FunctionTest.h @@ -110,6 +110,7 @@ class Compare2Function { function2_(FunctionBase::funcRegistrar_.createByType(name2)) { function1_->init(config); function2_->init(config); + initArgsCallback_ = nullptr; } ~Compare2Function() {} @@ -170,6 +171,10 @@ class Compare2Function { *seq2_)); } + void registerInitCallback(std::function callback) { + initArgsCallback_ = callback; + } + // output need only contains shape, do not contains data. void addOutputs(const BufferArg& output, ArgType argType = ASSIGN_TO) { size_t size = @@ -340,6 +345,10 @@ class Compare2Function { initArg(*func1Inputs_[i]); } + if (initArgsCallback_ != nullptr) { + initArgsCallback_(*func1Inputs_[i], i); + } + copyArg_(*func1Inputs_[i], *func2Inputs_[i]); } } @@ -386,6 +395,7 @@ class Compare2Function { std::shared_ptr seq1_; std::shared_ptr seq2_; test::CopyArgument copyArg_; + std::function initArgsCallback_; }; class CpuGpuFuncCompare diff --git a/paddle/function/ScaleSubRegionOp.cpp b/paddle/function/ScaleSubRegionOp.cpp new file mode 100644 index 0000000000000..a080505d7df83 --- /dev/null +++ b/paddle/function/ScaleSubRegionOp.cpp @@ -0,0 +1,155 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ScaleSubRegionOp.h" +#include "paddle/function/TensorShape.h" + +namespace paddle { + +template <> +void ScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + memcpy(outputs, inputs, number * channel * height * width * sizeof(real)); + + for (int n = 0; n < number; ++n) { + // indices start from 1 + int offset = n * 6; + for (int c = indices[offset] - 1; c < indices[offset + 1]; ++c) { + for (int h = indices[offset + 2] - 1; h < indices[offset + 3]; ++h) { + for (int w = indices[offset + 4] - 1; w < indices[offset + 5]; ++w) { + int idx = ((n * channel + c) * height + h) * width + w; + outputs[idx] *= value; + } + } + } + } +} + +template <> +void ScaleSubRegionGrad(const real* inGrad, + real* outGrad, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + for (int n = 0; n < number; ++n) { + for (int c = 0; c < channel; ++c) { + for (int h = 0; h < height; ++h) { + for (int w = 0; w < width; ++w) { + int idx = ((n * channel + c) * height + h) * width + w; + int offset = n * 6; + if (c >= (indices[offset] - 1) && c <= (indices[offset + 1] - 1) && + h >= (indices[offset + 2] - 1) && + h <= (indices[offset + 3] - 1) && + w >= (indices[offset + 4] - 1) && + w <= (indices[offset + 5] - 1)) { + outGrad[idx] += inGrad[idx] * value; + } else { + outGrad[idx] += inGrad[idx]; + } + } + } + } + } +} + +/** + * \brief For each instance, ScaleSubRegion can be used to multiply a value to + * a specified sub continuous region. By providing start index and end + * index for C/H/W, you can specify the location and shape of the region. + * + * Argument in this Function: + * \param inputs A 4-D tensor with shape [N, C, H, W], only one input. + * \param indices A 2-D tensor with shape [N, 6], indicates the sub region. + * \param outputs A 4-D tensor with same shape as inputs, output value. + */ +template +class ScaleSubRegionFunc : public FunctionBase { +public: + void init(const FuncConfig& config) override { conf_ = config; } + + void calc(const BufferArgs& inputs, const BufferArgs& outputs) override { + CHECK_EQ(2UL, inputs.size()); + CHECK_EQ(1UL, outputs.size()); + CHECK_EQ(outputs[0].getArgType(), ASSIGN_TO); + + TensorShape shape = inputs[0].shape(); + + ScaleSubRegion(outputs[0].data(), + inputs[0].data(), + inputs[1].data(), + shape, + conf_); + } + +private: + FuncConfig conf_; +}; + +/** + * \brief The backward propagation of ScaleSubRegion Function. + * + * Argument in this Function: + * \param inputs A 4-D tensor with shape [N, C, H, W], output gradient. + * \param indices A 2-D tensor with shape [N, 6], indicates the sub region. + * \param outputs A 4-D tensor with shape [N, C, H, W], gradient of input value. + */ + +template +class ScaleSubRegionGradFunc : public FunctionBase { +public: + void init(const FuncConfig& config) override { conf_ = config; } + + void calc(const BufferArgs& inputs, const BufferArgs& outputs) override { + CHECK_EQ(2UL, inputs.size()); + CHECK_EQ(1UL, outputs.size()); + CHECK_EQ(outputs[0].getArgType(), ADD_TO); + + TensorShape shape = inputs[0].shape(); + + ScaleSubRegionGrad(inputs[0].data(), + outputs[0].data(), + inputs[1].data(), + shape, + conf_); + } + +private: + FuncConfig conf_; +}; + +REGISTER_TYPED_FUNC(ScaleSubRegion, CPU, ScaleSubRegionFunc); +REGISTER_TYPED_FUNC(ScaleSubRegionGrad, CPU, ScaleSubRegionGradFunc); +#ifdef PADDLE_WITH_CUDA +REGISTER_TYPED_FUNC(ScaleSubRegion, GPU, ScaleSubRegionFunc); +REGISTER_TYPED_FUNC(ScaleSubRegionGrad, GPU, ScaleSubRegionGradFunc); +#endif + +} // namespace paddle diff --git a/paddle/function/ScaleSubRegionOp.h b/paddle/function/ScaleSubRegionOp.h new file mode 100644 index 0000000000000..0480c8577f3fb --- /dev/null +++ b/paddle/function/ScaleSubRegionOp.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Function.h" + +namespace paddle { + +/** + * \brief Function to multiply a value to values in specified sub continuous + * region. Indices must be provided to indcate the location and shape of + * the region and the multiplied value is passed by configure variable. + * + * + * \param[out] outputs Output value. + * \param[in] inputs Input data which contains NCHW information. + * \param[in] indices Indices data to indcate the sub region. + * \param[in] shape Tensor shape of input value. + * \param[in] conf Configure variable which contains the multiplied value. + */ +template +void ScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + const TensorShape shape, + const FuncConfig& conf); + +/** + * \brief Backward propagation function of ScaleSubRegion. + * + * \param[out] inGrad Gradients of previous layer. + * \param[in] outGrad Output gradient. + * \param[in] indices Indices data. + * \param[in] shape The Shape of input tensor. + * \param[in] conf Configure variable. + */ +template +void ScaleSubRegionGrad(const real* inGrad, + real* outGrad, + const real* indices, + const TensorShape shape, + const FuncConfig& conf); +} // namespace paddle diff --git a/paddle/function/ScaleSubRegionOpGpu.cu b/paddle/function/ScaleSubRegionOpGpu.cu new file mode 100644 index 0000000000000..8aae2e44c3fdc --- /dev/null +++ b/paddle/function/ScaleSubRegionOpGpu.cu @@ -0,0 +1,116 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ScaleSubRegionOp.h" +#include "hl_base.h" + +namespace paddle { + +__global__ void KeScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + real value, + int channel, + int height, + int width, + int nthreads) { + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < nthreads) { + const int w = idx % width; + const int h = (idx / width) % height; + const int c = (idx / width / height) % channel; + const int n = idx / width / height / channel; + + const int offset = n * 6; + if (c >= (indices[offset] - 1) && c <= (indices[offset + 1] - 1) && + h >= (indices[offset + 2] - 1) && h <= (indices[offset + 3] - 1) && + w >= (indices[offset + 4] - 1) && w <= (indices[offset + 5] - 1)) { + outputs[idx] = inputs[idx] * value; + } else { + outputs[idx] = inputs[idx]; + } + } +} + +template <> +void ScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + size_t nth = number * channel * height * width; + int blockSize = 1024; + int gridSize = (nth + blockSize - 1) / blockSize; + + KeScaleSubRegion<<>>( + outputs, inputs, indices, value, channel, height, width, nth); + CHECK_SYNC("ScaleSubRegion"); +} + +__global__ void KeScaleSubRegionDiff(const real* inGrad, + real* outGrad, + const real* indices, + real value, + int channel, + int height, + int width, + int nthreads) { + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < nthreads) { + const int w = idx % width; + const int h = (idx / width) % height; + const int c = (idx / width / height) % channel; + const int n = idx / width / height / channel; + + const int offset = n * 6; + if (c >= (indices[offset] - 1) && c <= (indices[offset + 1] - 1) && + h >= (indices[offset + 2] - 1) && h <= (indices[offset + 3] - 1) && + w >= (indices[offset + 4] - 1) && w <= (indices[offset + 5] - 1)) { + outGrad[idx] += inGrad[idx] * value; + } else { + outGrad[idx] += inGrad[idx]; + } + } +} + +template <> +void ScaleSubRegionGrad(const real* inGrad, + real* outGrad, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + size_t nth = number * channel * height * width; + int blockSize = 1024; + int gridSize = (nth + blockSize - 1) / blockSize; + + KeScaleSubRegionDiff<<>>( + inGrad, outGrad, indices, value, channel, height, width, nth); + CHECK_SYNC("ScaleSubRegionGrad"); +} + +} // namespace paddle diff --git a/paddle/function/ScaleSubRegionOpTest.cpp b/paddle/function/ScaleSubRegionOpTest.cpp new file mode 100644 index 0000000000000..43331f258ddda --- /dev/null +++ b/paddle/function/ScaleSubRegionOpTest.cpp @@ -0,0 +1,72 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include "FunctionTest.h" + +namespace paddle { + +TEST(ScaleSubRegion, real) { + for (size_t numSamples : {5, 32}) { + for (size_t channels : {5, 32}) { + for (size_t imgSizeH : {5, 33}) { + for (size_t imgSizeW : {5, 32}) { + for (real value : {-0.5, 0.0, 0.5}) { + for (bool firstHalf : {false, true}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW; + + for (bool testGrad : {false, true}) { + CpuGpuFuncCompare compare( + testGrad ? "ScaleSubRegionGrad" : "ScaleSubRegion", + FuncConfig().set("value", value)); + + TensorShape shape{numSamples, channels, imgSizeH, imgSizeW}; + TensorShape indicesShape{numSamples, 6}; + + compare.addInputs(BufferArg(VALUE_TYPE_FLOAT, shape)); + compare.addInputs(BufferArg(VALUE_TYPE_FLOAT, indicesShape)); + + compare.registerInitCallback([=](BufferArg& arg, size_t index) { + if (index == 1) { + real* data = (real*)arg.data(); + + for (size_t i = 0; i < numSamples; ++i) { + size_t offset = i * 6; + data[offset] = firstHalf ? 1 : channels / 2; + data[offset + 1] = firstHalf ? channels / 2 : channels; + data[offset + 2] = firstHalf ? 1 : imgSizeH / 2; + data[offset + 3] = firstHalf ? imgSizeH / 2 : imgSizeH; + data[offset + 4] = firstHalf ? 1 : imgSizeW / 2; + data[offset + 5] = firstHalf ? imgSizeW / 2 : imgSizeW; + } + } + }); + + compare.addOutputs( + BufferArg( + VALUE_TYPE_FLOAT, shape, testGrad ? ADD_TO : ASSIGN_TO), + testGrad ? ADD_TO : ASSIGN_TO); + compare.run(); + } + } + } + } + } + } + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNAddtoLayer.cpp b/paddle/gserver/layers/MKLDNNAddtoLayer.cpp index 8eb700723f2cf..0f2b67fd758ec 100644 --- a/paddle/gserver/layers/MKLDNNAddtoLayer.cpp +++ b/paddle/gserver/layers/MKLDNNAddtoLayer.cpp @@ -54,7 +54,6 @@ void MKLDNNAddtoLayer::reshape( ow = iw; reshapeOutput(oh, ow); resizeOutput(bs, oc * oh * ow); - printSizeInfo(); } void MKLDNNAddtoLayer::resetFwd(std::vector& pipeline, @@ -62,16 +61,14 @@ void MKLDNNAddtoLayer::resetFwd(std::vector& pipeline, MKLDNNMatrixPtr& wgt, MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out) { - if (biases_) { - LOG(FATAL) << "not implemented yet"; - } - resetFwdBuffers(inVals_, out); + resetFwdBuffers(inVals_, bias, out); in = inVals_[0]; std::shared_ptr fwdPD; - resetFwdPD(fwdPD, inVals_, out); + std::shared_ptr biasPD; + resetFwdPD(fwdPD, biasPD, inVals_, bias, out); - resetFwdPipeline(pipeline, fwdPD, inVals_, out); + resetFwdPipeline(pipeline, fwdPD, biasPD, inVals_, bias, out); } void MKLDNNAddtoLayer::resetBwd(std::vector& pipeline, @@ -79,7 +76,7 @@ void MKLDNNAddtoLayer::resetBwd(std::vector& pipeline, MKLDNNMatrixPtr& wgt, MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out) { - resetBwdBuffers(inGrads_, out); + resetBwdBuffers(inGrads_, bias, out); in = inGrads_[0]; // backward only need share output grad to input grad @@ -89,6 +86,20 @@ void MKLDNNAddtoLayer::resetBwd(std::vector& pipeline, inputLayers_[i]->getOutputGrad()->setData(inGrads_[i]->getData()); } } + + // backward bias + bwdBias_ = nullptr; + if (bias) { + std::vector scales(bs_, 1.0); + std::vector srcPDs(bs_, bias->getPrimitiveDesc()); + auto biasPD = sum::primitive_desc(bias->getMemoryDesc(), scales, srcPDs); + std::vector srcs; + for (size_t i = 0; i < grads_.size(); ++i) { + srcs.push_back(*(grads_[i])); + } + bwdBias_.reset(new sum(biasPD, srcs, *bias)); + pipeline.push_back(*bwdBias_); + } } void MKLDNNAddtoLayer::updateWeights(const UpdateCallback& callback) { @@ -97,7 +108,25 @@ void MKLDNNAddtoLayer::updateWeights(const UpdateCallback& callback) { } } +void MKLDNNAddtoLayer::prepareBias(MKLDNNMatrixPtr& bias, + const MatrixPtr& biasMat, + const MKLDNNMatrixPtr& out, + std::vector& outs) { + auto pd = MKLDNNMatrix::createPrimitiveDesc( + {(int)layerSize_}, memory::format::x, engine_); + bias = MKLDNNMatrix::create(pd, biasMat); + outs.clear(); + real* data = out->getData(); + CHECK_EQ(bs_ * layerSize_, out->getElementCnt()); + for (int i = 0; i < bs_; ++i) { + MatrixPtr tmp = + Matrix::create(data + i * layerSize_, 1, layerSize_, false, false); + outs.push_back(MKLDNNMatrix::create(bias->getPrimitiveDesc(), tmp)); + } +} + void MKLDNNAddtoLayer::resetFwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out) { inputs.resize(inputLayers_.size()); for (size_t i = 0; i < inputs.size(); i++) { @@ -110,12 +139,20 @@ void MKLDNNAddtoLayer::resetFwdBuffers(std::vector& inputs, } resetOutValue(out, inputs[0]->getPrimitiveDesc()); + + if (biases_ && biases_->getW()) { + prepareBias(bias, biases_->getW(), out, vals_); + } else { + bias = nullptr; + } } void MKLDNNAddtoLayer::resetFwdPD(std::shared_ptr& pd, + std::shared_ptr& biasPD, std::vector& inputs, + MKLDNNMatrixPtr bias, MKLDNNMatrixPtr out) { - std::vector scales(inputs.size(), 1.0); + std::vector scales(inputs.size(), 1.0); std::vector srcPDs; for (size_t i = 0; i < inputs.size(); i++) { srcPDs.push_back(inputs[i]->getPrimitiveDesc()); @@ -123,12 +160,23 @@ void MKLDNNAddtoLayer::resetFwdPD(std::shared_ptr& pd, CHECK(out); pd.reset(new sum::primitive_desc(out->getMemoryDesc(), scales, srcPDs)); CHECK_PRIMITIVE_DESC_EQ(out, pd->dst_primitive_desc()); + + biasPD = nullptr; + if (bias) { + std::vector scales(2, 1.0); + std::vector srcPDs(2, bias->getPrimitiveDesc()); + biasPD.reset( + new sum::primitive_desc(bias->getMemoryDesc(), scales, srcPDs)); + CHECK_PRIMITIVE_DESC_EQ(bias, biasPD->dst_primitive_desc()); + } } void MKLDNNAddtoLayer::resetFwdPipeline( std::vector& pipeline, std::shared_ptr& pd, + std::shared_ptr& biasPD, std::vector& inputs, + MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out) { std::vector srcs; for (size_t i = 0; i < inputs.size(); i++) { @@ -136,9 +184,23 @@ void MKLDNNAddtoLayer::resetFwdPipeline( } fwd_.reset(new sum(*pd, srcs, *out)); pipeline.push_back(*fwd_); + + fwdBias_.clear(); + if (biasPD == nullptr || bias == nullptr) { + return; + } + fwdBias_.resize(vals_.size()); + for (size_t i = 0; i < vals_.size(); ++i) { + std::vector srcs; + srcs.push_back(*(vals_[i])); + srcs.push_back(*bias); + fwdBias_[i].reset(new sum(*biasPD, srcs, *vals_[i])); + pipeline.push_back(*fwdBias_[i]); + } } void MKLDNNAddtoLayer::resetBwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out) { CHECK(outVal_); resetOutGrad(out, outVal_->getPrimitiveDesc()); @@ -149,6 +211,12 @@ void MKLDNNAddtoLayer::resetBwdBuffers(std::vector& inputs, resetInGrad(inputs[i], inVal_->getPrimitiveDesc(), i); CHECK_PRIMITIVE_DESC_EQ(inputs[i], out->getPrimitiveDesc()); } + + if (biases_ && biases_->getWGrad()) { + prepareBias(bias, biases_->getWGrad(), out, grads_); + } else { + bias = nullptr; + } } } // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNAddtoLayer.h b/paddle/gserver/layers/MKLDNNAddtoLayer.h index 15f74ec5bdf3d..24504b7b4f507 100644 --- a/paddle/gserver/layers/MKLDNNAddtoLayer.h +++ b/paddle/gserver/layers/MKLDNNAddtoLayer.h @@ -32,9 +32,15 @@ class MKLDNNAddtoLayer : public MKLDNNLayer { // layer size == ic * ih * iw == oc * oh *ow, and can not be changed size_t layerSize_; - // TODO(TJ): this part has not been optimized by MKL-DNN std::unique_ptr biases_; + // buffers for adding bias + std::vector vals_; + std::vector grads_; + // primitives for adding bias + std::vector> fwdBias_; + std::shared_ptr bwdBias_; + public: explicit MKLDNNAddtoLayer(const LayerConfig& config) : MKLDNNLayer(config) {} @@ -91,20 +97,34 @@ class MKLDNNAddtoLayer : public MKLDNNLayer { * reset pipeline. */ void resetFwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out); void resetFwdPD(std::shared_ptr& pd, + std::shared_ptr& biasPD, std::vector& inputs, + MKLDNNMatrixPtr bias, MKLDNNMatrixPtr out); void resetFwdPipeline(std::vector& pipeline, std::shared_ptr& pd, + std::shared_ptr& biasPD, std::vector& inputs, + MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out); /** * Backward functions: reset buffers(inputs, output, bias) */ void resetBwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, MKLDNNMatrixPtr& out); + + /** + * prepare for bias + */ + void prepareBias(MKLDNNMatrixPtr& bias, + const MatrixPtr& biasMat, + const MKLDNNMatrixPtr& out, + std::vector& outs); }; } // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp b/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp index 9b0ae20f089e3..071bdf54d5dc9 100644 --- a/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp +++ b/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp @@ -119,13 +119,12 @@ void MKLDNNBatchNormLayer::reshape( int& bs, int& ic, int& ih, int& iw, int oc, int& oh, int& ow) { reshapeInput(bs, ih, iw); oh = ih; - ow = ow; + ow = iw; // ic_ and oc can not be changed CHECK_EQ(inputElemenCnt_ / bs / ih / iw, (size_t)ic) << "Input channel can not be changed"; reshapeOutput(oh, ow); resizeOutput(bs, oc * oh * ow); - printSizeInfo(); } void MKLDNNBatchNormLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/MKLDNNConvLayer.cpp b/paddle/gserver/layers/MKLDNNConvLayer.cpp index b8120eda1e2da..8aa54e0a9efa7 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.cpp +++ b/paddle/gserver/layers/MKLDNNConvLayer.cpp @@ -102,8 +102,6 @@ void MKLDNNConvLayer::reshape( reshapeOutput(oh, ow); resizeOutput(bs, oc * oh * ow); - - printSizeInfo(); } void MKLDNNConvLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/MKLDNNConvLayer.h b/paddle/gserver/layers/MKLDNNConvLayer.h index 1fed0e1c6565b..9c69136684e5f 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.h +++ b/paddle/gserver/layers/MKLDNNConvLayer.h @@ -92,7 +92,7 @@ class MKLDNNConvLayer : public MKLDNNLayer { void printSizeInfo() override { MKLDNNLayer::printSizeInfo(); VLOG(MKLDNN_SIZES) << getName() << ": fh: " << fh_ << ", fw: " << fw_ - << ": ph: " << ph_ << ", pw: " << pw_ << ", sh: " << sh_ + << ", ph: " << ph_ << ", pw: " << pw_ << ", sh: " << sh_ << ", sw: " << sw_ << ", dh: " << dh_ << ", dw: " << dw_; } diff --git a/paddle/gserver/layers/MKLDNNFcLayer.cpp b/paddle/gserver/layers/MKLDNNFcLayer.cpp index 3429c53d2396e..350ec65fffbc7 100644 --- a/paddle/gserver/layers/MKLDNNFcLayer.cpp +++ b/paddle/gserver/layers/MKLDNNFcLayer.cpp @@ -84,8 +84,6 @@ void MKLDNNFcLayer::reshape( reshapeOutput(oh, ow); resizeOutput(bs, oc); - - printSizeInfo(); } void MKLDNNFcLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/MKLDNNLayer.cpp b/paddle/gserver/layers/MKLDNNLayer.cpp index 82ef344c7b2aa..e75ac5ba4647a 100644 --- a/paddle/gserver/layers/MKLDNNLayer.cpp +++ b/paddle/gserver/layers/MKLDNNLayer.cpp @@ -287,7 +287,7 @@ void MKLDNNLayer::resetMergeGrad(MKLDNNMatrixPtr& out) { return; } CHECK(out) << "should have reset internal ouput grad"; - std::vector scales(outputMap_.size(), 1.0); + std::vector scales(outputMap_.size(), 1.0); std::vector srcPDs; std::vector srcs; for (auto it = outputMap_.begin(); it != outputMap_.end(); ++it) { diff --git a/paddle/gserver/layers/MKLDNNPoolLayer.cpp b/paddle/gserver/layers/MKLDNNPoolLayer.cpp index 6e89260f49979..a18c455beab96 100644 --- a/paddle/gserver/layers/MKLDNNPoolLayer.cpp +++ b/paddle/gserver/layers/MKLDNNPoolLayer.cpp @@ -71,8 +71,6 @@ void MKLDNNPoolLayer::reshape( reshapeOutput(oh, ow); resizeOutput(bs, oc * oh * ow); - - printSizeInfo(); } void MKLDNNPoolLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/ROIPoolLayer.cpp b/paddle/gserver/layers/ROIPoolLayer.cpp new file mode 100644 index 0000000000000..35d4b12d3d357 --- /dev/null +++ b/paddle/gserver/layers/ROIPoolLayer.cpp @@ -0,0 +1,220 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ROIPoolLayer.h" + +namespace paddle { + +REGISTER_LAYER(roi_pool, ROIPoolLayer); + +bool ROIPoolLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + + const ROIPoolConfig& layerConf = config_.inputs(0).roi_pool_conf(); + pooledWidth_ = layerConf.pooled_width(); + pooledHeight_ = layerConf.pooled_height(); + spatialScale_ = layerConf.spatial_scale(); + + return true; +} + +void ROIPoolLayer::forward(PassType passType) { + Layer::forward(passType); + + const ROIPoolConfig& layerConf = config_.inputs(0).roi_pool_conf(); + height_ = getInput(0).getFrameHeight(); + if (!height_) height_ = layerConf.height(); + width_ = getInput(0).getFrameWidth(); + if (!width_) width_ = layerConf.width(); + channels_ = getInputValue(0)->getWidth() / width_ / height_; + + size_t batchSize = getInput(0).getBatchSize(); + size_t numROIs = getInput(1).getBatchSize(); + + MatrixPtr dataValue = getInputValue(0); + MatrixPtr roiValue = getInputValue(1); + resetOutput(numROIs, channels_ * pooledHeight_ * pooledWidth_); + MatrixPtr outputValue = getOutputValue(); + + if (useGpu_) { // TODO(guosheng): implement on GPU later + MatrixPtr dataCpuBuffer; + Matrix::resizeOrCreate(dataCpuBuffer, + dataValue->getHeight(), + dataValue->getWidth(), + false, + false); + MatrixPtr roiCpuBuffer; + Matrix::resizeOrCreate(roiCpuBuffer, + roiValue->getHeight(), + roiValue->getWidth(), + false, + false); + dataCpuBuffer->copyFrom(*dataValue); + roiCpuBuffer->copyFrom(*roiValue); + dataValue = dataCpuBuffer; + roiValue = roiCpuBuffer; + MatrixPtr outputCpuBuffer; + Matrix::resizeOrCreate(outputCpuBuffer, + outputValue->getHeight(), + outputValue->getWidth(), + false, + false); + outputCpuBuffer->copyFrom(*outputValue); + outputValue = outputCpuBuffer; + } + + real* bottomData = dataValue->getData(); + size_t batchOffset = dataValue->getWidth(); + size_t channelOffset = height_ * width_; + real* bottomROIs = roiValue->getData(); + size_t roiOffset = roiValue->getWidth(); + size_t poolChannelOffset = pooledHeight_ * pooledWidth_; + + real* outputData = outputValue->getData(); + Matrix::resizeOrCreate(maxIdxs_, + numROIs, + channels_ * pooledHeight_ * pooledWidth_, + false, + false); + real* argmaxData = maxIdxs_->getData(); + + for (size_t n = 0; n < numROIs; ++n) { + // the first five elememts of each RoI should be: + // batch_idx, roi_x_start, roi_y_start, roi_x_end, roi_y_end + size_t roiBatchIdx = bottomROIs[0]; + size_t roiStartW = round(bottomROIs[1] * spatialScale_); + size_t roiStartH = round(bottomROIs[2] * spatialScale_); + size_t roiEndW = round(bottomROIs[3] * spatialScale_); + size_t roiEndH = round(bottomROIs[4] * spatialScale_); + CHECK_GE(roiBatchIdx, 0UL); + CHECK_LT(roiBatchIdx, batchSize); + size_t roiHeight = std::max(roiEndH - roiStartH + 1, 1UL); + size_t roiWidth = std::max(roiEndW - roiStartW + 1, 1UL); + real binSizeH = + static_cast(roiHeight) / static_cast(pooledHeight_); + real binSizeW = + static_cast(roiWidth) / static_cast(pooledWidth_); + real* batchData = bottomData + batchOffset * roiBatchIdx; + for (size_t c = 0; c < channels_; ++c) { + for (size_t ph = 0; ph < pooledHeight_; ++ph) { + for (size_t pw = 0; pw < pooledWidth_; ++pw) { + size_t hstart = static_cast(std::floor(ph * binSizeH)); + size_t wstart = static_cast(std::floor(pw * binSizeW)); + size_t hend = static_cast(std::ceil((ph + 1) * binSizeH)); + size_t wend = static_cast(std::ceil((pw + 1) * binSizeW)); + hstart = std::min(std::max(hstart + roiStartH, 0UL), height_); + wstart = std::min(std::max(wstart + roiStartW, 0UL), width_); + hend = std::min(std::max(hend + roiStartH, 0UL), height_); + wend = std::min(std::max(wend + roiStartW, 0UL), width_); + + bool isEmpty = (hend <= hstart) || (wend <= wstart); + size_t poolIndex = ph * pooledWidth_ + pw; + if (isEmpty) { + outputData[poolIndex] = 0; + argmaxData[poolIndex] = -1; + } + + for (size_t h = hstart; h < hend; ++h) { + for (size_t w = wstart; w < wend; ++w) { + size_t index = h * width_ + w; + if (batchData[index] > outputData[poolIndex]) { + outputData[poolIndex] = batchData[index]; + argmaxData[poolIndex] = index; + } + } + } + } + } + batchData += channelOffset; + outputData += poolChannelOffset; + argmaxData += poolChannelOffset; + } + bottomROIs += roiOffset; + } + if (useGpu_) { + getOutputValue()->copyFrom(*outputValue); + } +} + +void ROIPoolLayer::backward(const UpdateCallback& callback) { + MatrixPtr inGradValue = getInputGrad(0); + MatrixPtr outGradValue = getOutputGrad(); + MatrixPtr roiValue = getInputValue(1); + + if (useGpu_) { + MatrixPtr inGradCpuBuffer; + Matrix::resizeOrCreate(inGradCpuBuffer, + inGradValue->getHeight(), + inGradValue->getWidth(), + false, + false); + MatrixPtr outGradCpuBuffer; + Matrix::resizeOrCreate(outGradCpuBuffer, + outGradValue->getHeight(), + outGradValue->getWidth(), + false, + false); + MatrixPtr roiCpuBuffer; + Matrix::resizeOrCreate(roiCpuBuffer, + roiValue->getHeight(), + roiValue->getWidth(), + false, + false); + inGradCpuBuffer->copyFrom(*inGradValue); + outGradCpuBuffer->copyFrom(*outGradValue); + roiCpuBuffer->copyFrom(*roiValue); + inGradValue = inGradCpuBuffer; + outGradValue = outGradCpuBuffer; + roiValue = roiCpuBuffer; + } + + real* bottomROIs = roiValue->getData(); + size_t numROIs = getInput(1).getBatchSize(); + size_t roiOffset = getInputValue(1)->getWidth(); + + real* inDiffData = inGradValue->getData(); + size_t batchOffset = getInputValue(0)->getWidth(); + size_t channelOffset = height_ * width_; + + real* outDiffData = outGradValue->getData(); + size_t poolChannelOffset = pooledHeight_ * pooledWidth_; + real* argmaxData = maxIdxs_->getData(); + + for (size_t n = 0; n < numROIs; ++n) { + size_t roiBatchIdx = bottomROIs[0]; + real* batchDiffData = inDiffData + batchOffset * roiBatchIdx; + for (size_t c = 0; c < channels_; ++c) { + for (size_t ph = 0; ph < pooledHeight_; ++ph) { + for (size_t pw = 0; pw < pooledWidth_; ++pw) { + size_t poolIndex = ph * pooledWidth_ + pw; + if (argmaxData[poolIndex] > 0) { + size_t index = static_cast(argmaxData[poolIndex]); + batchDiffData[index] += outDiffData[poolIndex]; + } + } + } + batchDiffData += channelOffset; + outDiffData += poolChannelOffset; + argmaxData += poolChannelOffset; + } + bottomROIs += roiOffset; + } + + if (useGpu_) { + getInputGrad(0)->copyFrom(*inGradValue); + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ROIPoolLayer.h b/paddle/gserver/layers/ROIPoolLayer.h new file mode 100644 index 0000000000000..4f07e49d6fd1e --- /dev/null +++ b/paddle/gserver/layers/ROIPoolLayer.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" + +namespace paddle { + +/** + * A layer used by Fast R-CNN to extract feature maps of ROIs from the last + * feature map. + * - Input: This layer needs two input layers: The first input layer is a + * convolution layer; The second input layer contains the ROI data + * which is the output of ProposalLayer in Faster R-CNN. layers for + * generating bbox location offset and the classification confidence. + * - Output: The ROIs' feature map. + * Reference: + * Shaoqing Ren, Kaiming He, Ross Girshick, and Jian Sun. + * Faster R-CNN: Towards Real-Time Object Detection with Region Proposal + * Networks + */ + +class ROIPoolLayer : public Layer { +protected: + size_t channels_; + size_t width_; + size_t height_; + size_t pooledWidth_; + size_t pooledHeight_; + real spatialScale_; + + // Since there is no int matrix, use real maxtrix instead. + MatrixPtr maxIdxs_; + +public: + explicit ROIPoolLayer(const LayerConfig& config) : Layer(config) {} + + bool init(const LayerMap& layerMap, + const ParameterMap& parameterMap) override; + + void forward(PassType passType) override; + void backward(const UpdateCallback& callback = nullptr) override; +}; +} // namespace paddle diff --git a/paddle/gserver/layers/ScaleSubRegionLayer.cpp b/paddle/gserver/layers/ScaleSubRegionLayer.cpp new file mode 100644 index 0000000000000..aa6778aef4e89 --- /dev/null +++ b/paddle/gserver/layers/ScaleSubRegionLayer.cpp @@ -0,0 +1,78 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ScaleSubRegionLayer.h" +#include "paddle/utils/Stat.h" +namespace paddle { + +REGISTER_LAYER(scale_sub_region, ScaleSubRegionLayer); + +bool ScaleSubRegionLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + CHECK_EQ(static_cast(inputLayers_.size()), 2); + auto& conf = config_.inputs(0).scale_sub_region_conf(); + value_ = conf.value(); + + createFunction(forward_, "ScaleSubRegion", FuncConfig().set("value", value_)); + createFunction( + backward_, "ScaleSubRegionGrad", FuncConfig().set("value", value_)); + + return true; +} + +void ScaleSubRegionLayer::forward(PassType passType) { + Layer::forward(passType); + auto in0 = getInput(0); + imgH_ = in0.getFrameHeight(); + imgW_ = in0.getFrameWidth(); + if (imgH_ == 0 || imgW_ == 0) { + auto& conf = config_.inputs(0).scale_sub_region_conf(); + imgH_ = conf.image_conf().img_size_y(); + imgW_ = conf.image_conf().img_size(); + } + MatrixPtr imgV = in0.value; + size_t batchSize = imgV->getHeight(); + size_t spatialSize = imgH_ * imgW_; + channelsNum_ = imgV->getWidth() / spatialSize; + shape_ = TensorShape({batchSize, channelsNum_, imgH_, imgW_}); + + resetOutput(batchSize, imgV->getWidth()); + auto& out = getOutput(); + out.setFrameHeight(imgH_); + out.setFrameWidth(imgW_); + + MatrixPtr indicesV = getInputValue(1); + indicesShape_ = TensorShape({batchSize, 6}); + + REGISTER_TIMER_INFO("ScaleSubRegionForward", getName().c_str()); + BufferArgs inArgs; + BufferArgs outArgs; + inArgs.addArg(*imgV, shape_); + inArgs.addArg(*indicesV, indicesShape_); + outArgs.addArg(*out.value, shape_, ASSIGN_TO); + forward_[0]->calc(inArgs, outArgs); +} + +void ScaleSubRegionLayer::backward(const UpdateCallback& callback) { + REGISTER_TIMER_INFO("ScaleSubRegionBackward", getName().c_str()); + BufferArgs inArgs; + BufferArgs outArgs; + inArgs.addArg(*getOutputGrad(), shape_); + inArgs.addArg(*getInputValue(1), indicesShape_); + outArgs.addArg(*getInputGrad(0), shape_, ADD_TO); + backward_[0]->calc(inArgs, outArgs); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ScaleSubRegionLayer.h b/paddle/gserver/layers/ScaleSubRegionLayer.h new file mode 100644 index 0000000000000..a27c56de93bb6 --- /dev/null +++ b/paddle/gserver/layers/ScaleSubRegionLayer.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" + +namespace paddle { + +/** + * \brief For each instance, this layer can be used to multiply a value to a + * specified sub continuous region. By providing start index and end + * index for C/H/W, you can specify the location and shape of the + * region. + * + * input_0: Input value. + * input_1: Indices value to specify the location an shape of the + * region. + */ +class ScaleSubRegionLayer : public Layer { +public: + explicit ScaleSubRegionLayer(const LayerConfig& config) : Layer(config) {} + + ~ScaleSubRegionLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + + void backward(const UpdateCallback& callback = nullptr); + +protected: + TensorShape shape_; + TensorShape indicesShape_; + size_t imgH_; + size_t imgW_; + size_t channelsNum_; + real value_; +}; + +} // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 1a46fb49153a0..fcbcb5b0f1f4c 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -53,7 +53,7 @@ TEST(Operator, dot_mul) { TEST(Projection, context) { for (auto contextStart : {-5, -3, -1, 0, 3}) { for (auto contextLength : {1, 2, 5, 7}) { - for (auto batchSize : {1, 2, 5, 20, 50}) { + for (auto batchSize : {1, 2, 5, 20}) { for (auto trainablePadding : {false, true}) { LOG(INFO) << " contextStart=" << contextStart << " contextLength=" << contextLength @@ -585,14 +585,14 @@ TEST(Layer, maxoutLayer) { } void testFcLayer(string format, size_t nnz) { TestConfig config; - config.biasSize = 4096; + config.biasSize = 1024; config.layerConfig.set_type("fc"); - config.layerConfig.set_size(4096); + config.layerConfig.set_size(1024); config.layerConfig.set_active_type("sigmoid"); config.layerConfig.set_drop_rate(0.1); config.inputDefs.push_back( - {INPUT_DATA, "layer_0", 8192, nnz, ParaSparse(format)}); + {INPUT_DATA, "layer_0", 2048, nnz, ParaSparse(format)}); config.layerConfig.add_inputs(); LOG(INFO) << config.inputDefs[0].sparse.sparse << " " @@ -609,9 +609,9 @@ void testFcLayer(string format, size_t nnz) { } TEST(Layer, fcLayer) { - testFcLayer("", 4096 * 4096 * 2); - testFcLayer("csc", 4096 * 40); - testFcLayer("csr", 4096 * 40); + testFcLayer("", 1024 * 1024 * 2); + testFcLayer("csc", 1024 * 10); + testFcLayer("csr", 1024 * 10); } TEST(Layer, SelectiveFullyConnectedLayer) { @@ -1995,7 +1995,7 @@ TEST(Layer, multibox_loss) { TEST(Layer, TransLayer) { TestConfig config; const int height = 128; - const int width = 1028; + const int width = 256; config.layerConfig.set_type("trans"); config.layerConfig.set_size(width); @@ -2056,6 +2056,43 @@ TEST(Layer, CropLayer) { } } +TEST(Layer, roi_pool) { + TestConfig config; + config.layerConfig.set_type("roi_pool"); + config.biasSize = 0; + LayerInputConfig* input = config.layerConfig.add_inputs(); + ROIPoolConfig* roiPoolConf = input->mutable_roi_pool_conf(); + roiPoolConf->set_pooled_width(7); + roiPoolConf->set_pooled_height(7); + roiPoolConf->set_spatial_scale(1. / 16); + roiPoolConf->set_width(14); + roiPoolConf->set_height(14); + + const size_t roiNum = 10; + const size_t roiDim = 10; + const size_t batchSize = 5; + MatrixPtr roiValue = Matrix::create(roiNum, roiDim, false, false); + roiValue->zeroMem(); + real* roiData = roiValue->getData(); + for (size_t i = 0; i < roiNum; ++i) { + roiData[i * roiDim + 0] = std::rand() % batchSize; + roiData[i * roiDim + 1] = std::rand() % 224; // xMin + roiData[i * roiDim + 2] = std::rand() % 224; // yMin + size_t xMin = static_cast(roiData[i * roiDim + 1]); + size_t yMin = static_cast(roiData[i * roiDim + 2]); + roiData[i * roiDim + 3] = xMin + std::rand() % (224 - xMin); // xMax + roiData[i * roiDim + 4] = yMin + std::rand() % (224 - yMin); // yMax + } + + config.inputDefs.push_back({INPUT_DATA, "input", 3 * 14 * 14, {}}); + config.inputDefs.push_back({INPUT_SELF_DEFINE_DATA, "rois", roiValue, {}}); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "roi_pool", batchSize, false, useGpu, false); + } +} + TEST(Layer, SwitchOrderLayer) { TestConfig config; // config input_0 @@ -2358,6 +2395,38 @@ TEST(Layer, ScaleShiftLayer) { } } +TEST(Layer, ScaleSubRegionLayer) { + const size_t batchSize = 64; + const size_t size = 4096; + TestConfig config; + config.layerConfig.set_type("scale_sub_region"); + config.inputDefs.push_back({INPUT_DATA, "input", size, 0}); + MatrixPtr indicesV = Matrix::create(batchSize, 6, false, false); + auto* data = indicesV->getData(); + for (size_t i = 0; i < batchSize; ++i) { + data[i * 2] = 2; + data[i * 2 + 1] = 4; + data[i * 2 + 2] = 16; + data[i * 2 + 3] = 32; + data[i * 2 + 4] = 16; + data[i * 2 + 5] = 32; + } + config.inputDefs.push_back({INPUT_SELF_DEFINE_DATA, "indices", indicesV, {}}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + ScaleSubRegionConfig* scaleSubRegionConf = + input->mutable_scale_sub_region_conf(); + ImageConfig* imgConf = scaleSubRegionConf->mutable_image_conf(); + imgConf->set_img_size(32); + imgConf->set_img_size_y(32); + imgConf->set_channels(4); + scaleSubRegionConf->set_value(2.0); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "scale_sub_region", batchSize, false, useGpu, false); + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/gserver/tests/test_MKLDNN.cpp b/paddle/gserver/tests/test_MKLDNN.cpp index 2e8d9f3333b36..a859e34c8996d 100644 --- a/paddle/gserver/tests/test_MKLDNN.cpp +++ b/paddle/gserver/tests/test_MKLDNN.cpp @@ -269,6 +269,7 @@ void testBatchNormLayer(const testBatchNormDesc& pm) { TEST(MKLDNNLayer, BatchNormLayer) { testBatchNormLayer({4, 10, 6, 6}); testBatchNormLayer({16, 32, 16, 16}); + testBatchNormLayer({4, 16, 8, 10}); } struct testImageDesc { @@ -296,17 +297,12 @@ static void getAddtoConfig(TestConfig& cfg, } void testAddtoLayer(const testImageDesc& pm, const size_t nInputs) { - CHECK_GE(nInputs, 1); + CHECK_GE(nInputs, 1UL); TestConfig dnnConfig; getAddtoConfig(dnnConfig, pm, nInputs); dnnConfig.layerConfig.set_type("mkldnn_addto"); - // TODO(TJ): test with bias - for (auto withBias : {false}) { - if (withBias) { - dnnConfig.biasSize = pm.ic * pm.ih * pm.iw; - } else { - dnnConfig.biasSize = 0; - } + for (auto withBias : {false, true}) { + dnnConfig.biasSize = withBias ? pm.ic * pm.ih * pm.iw : 0; RUN_MKLDNN_TEST_LAYER(dnnConfig, "addto", pm) } } diff --git a/paddle/math/MKLDNNMatrix.cpp b/paddle/math/MKLDNNMatrix.cpp index 21a8f73c3e650..a710479bab82e 100644 --- a/paddle/math/MKLDNNMatrix.cpp +++ b/paddle/math/MKLDNNMatrix.cpp @@ -152,12 +152,7 @@ void MKLDNNMatrix::downSpatial() { } memory::desc md = memory::desc(dstDims, getDtype(), dstFmt); memory::primitive_desc pd = memory::primitive_desc(md, getEngine()); - mkldnn_primitive_t result; - mkldnn::error::wrap_c_api( - mkldnn_primitive_create(&result, pd.get(), nullptr, nullptr), - "could not create a memory primitive"); - reset(result); - set_data_handle(data_); + resetMKLDNNMemory(pd, data_); } } // namespace paddle diff --git a/paddle/math/MKLDNNMatrix.h b/paddle/math/MKLDNNMatrix.h index 54cfefe23b3dc..39d40a1f61609 100644 --- a/paddle/math/MKLDNNMatrix.h +++ b/paddle/math/MKLDNNMatrix.h @@ -145,6 +145,27 @@ class MKLDNNMatrix : public CpuMatrix, public mkldnn::memory { m_.reset(); } + /** + * override the CpuMatrix::resize + */ + void resize(size_t newHeight, size_t newWidth) override { + m_->resize(newHeight, newWidth); + if (data_ == m_->getData() && elementCnt_ == newHeight * newWidth) { + return; + } + CpuMatrix::setData(data_); + height_ = newHeight; + width_ = newWidth; + elementCnt_ = newHeight * newWidth; + stride_ = width_; + auto pd = mkldnn::memory::primitive_desc( + mkldnn::memory::desc({(int)newHeight, (int)newWidth}, + getDtype(), + mkldnn::memory::format::nc), + getEngine()); + resetMKLDNNMemory(pd, data_); + } + /** * override Matrix::getData * check data before return @@ -215,6 +236,17 @@ class MKLDNNMatrix : public CpuMatrix, public mkldnn::memory { memory::format srcFmt, memory::format dstFmt, memory::dims dm); + /** + * reset this MKLDNN Memory from primitve desc + */ + void resetMKLDNNMemory(memory::primitive_desc pd, real* data) { + mkldnn_primitive_t result; + mkldnn::error::wrap_c_api( + mkldnn_primitive_create(&result, pd.get(), nullptr, nullptr), + "could not create a memory primitive"); + reset(result); + set_data_handle(data); + } private: // save the CpuMatrixPtr in case the buffer released outside diff --git a/paddle/math/MathFunctions.cpp b/paddle/math/MathFunctions.cpp index c2f17beeb8794..ba86eacbb5d53 100644 --- a/paddle/math/MathFunctions.cpp +++ b/paddle/math/MathFunctions.cpp @@ -206,7 +206,7 @@ double dotProduct(const int n, const double* x, const double* y) { } #endif -#if defined(PADDLE_USE_MKL) || defined(PADDLE_USE_MKLML) +#if defined(PADDLE_USE_MKLML) template <> void vExp(const int n, const float* a, float* r) { @@ -295,38 +295,6 @@ template void vAdd(const int n, const double* a, const double* b, double* r); #endif -#ifdef PADDLE_USE_MKL -template <> -void vInvSqrt(const int n, const float* a, float* r) { - vsInvSqrt(n, a, r); -} - -template <> -void vInvSqrt(const int n, const double* a, double* r) { - vdInvSqrt(n, a, r); -} - -template <> -void vLog1p(const int n, const float* a, float* r) { - vsLog1p(n, a, r); -} - -template <> -void vLog1p(const int n, const double* a, double* r) { - vdLog1p(n, a, r); -} - -template <> -void vTanh(const int n, const float* a, float* r) { - vsTanh(n, a, r); -} - -template <> -void vTanh(const int n, const double* a, double* r) { - vdTanh(n, a, r); -} -#else - DEFINE_MATRIX_BINARY_OP(vInvSqrt, b = 1.0f / std::sqrt(a)); template void vInvSqrt(const int n, const T* a, T* r) { @@ -357,6 +325,4 @@ template void vLog1p(const int n, const double* a, double* r); template void vTanh(const int n, const float* a, float* r); template void vTanh(const int n, const double* a, double* r); -#endif - } // namespace paddle diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index 8193aa4adffc0..f6e77029bdd75 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -21,11 +21,6 @@ limitations under the License. */ #include #endif -#ifdef PADDLE_USE_MKL -#include -#include -#endif - #if defined(PADDLE_USE_ATLAS) || defined(PADDLE_USE_VECLIB) extern "C" { #include diff --git a/paddle/math/tests/TensorCheck.h b/paddle/math/tests/TensorCheck.h index 5bc4a03067a75..b998e5772e70d 100644 --- a/paddle/math/tests/TensorCheck.h +++ b/paddle/math/tests/TensorCheck.h @@ -169,7 +169,7 @@ void TensorCheck(AssertEq compare, count++; } } - EXPECT_EQ(count, 0) << "There are " << count << " different element."; + EXPECT_EQ(count, 0) << "There are " << count << " different elements."; } template diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index eae87a5141ef1..709f7de2e4309 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -195,8 +195,13 @@ op_library(sequence_pool_op DEPS sequence_pooling) op_library(lstm_op DEPS sequence2batch lstm_compute) op_library(conv_transpose_op DEPS vol2col) op_library(gru_op DEPS sequence2batch gru_compute) -op_library(dynamic_recurrent_op SRCS dynamic_recurrent_op.cc rnn/recurrent_op_utils.cc - DEPS net_op tensor_array) +if(WITH_TESTING) + op_library(dynamic_recurrent_op SRCS dynamic_recurrent_op.cc rnn/recurrent_op_utils.cc + DEPS net_op tensor_array gtest) +else() + op_library(dynamic_recurrent_op SRCS dynamic_recurrent_op.cc rnn/recurrent_op_utils.cc + DEPS net_op tensor_array) +endif() op_library(recurrent_op SRCS recurrent_op.cc DEPS executor) list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) @@ -209,6 +214,7 @@ set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) +cc_test(beam_search_decode_op_test SRCS beam_search_decode_op_test.cc DEPS lod_tensor) cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory) cc_test(dynamic_recurrent_op_test SRCS dynamic_recurrent_op_test.cc rnn/recurrent_op_utils.cc diff --git a/paddle/operators/accuracy_op.cu b/paddle/operators/accuracy_op.cu index d0c4c0d25d6f4..1776f33105367 100644 --- a/paddle/operators/accuracy_op.cu +++ b/paddle/operators/accuracy_op.cu @@ -65,7 +65,7 @@ class AccuracyOpCUDAKernel : public framework::OpKernel { size_t num_samples = inference->dims()[0]; size_t infer_width = inference->dims()[1]; - cudaMemset((void**)&accuracy_data, 0, sizeof(float)); + PADDLE_ENFORCE(cudaMemset(accuracy_data, 0, sizeof(float))); if (num_samples == 0) { return; diff --git a/paddle/operators/accuracy_op.h b/paddle/operators/accuracy_op.h index e00d6c87e0519..d060e6edddb31 100644 --- a/paddle/operators/accuracy_op.h +++ b/paddle/operators/accuracy_op.h @@ -14,7 +14,6 @@ limitations under the License. */ #pragma once #include -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" namespace paddle { @@ -22,18 +21,6 @@ namespace operators { using Tensor = framework::Tensor; -template -using EigenMatrix = framework::EigenMatrix; - -template -using EigenVector = framework::EigenVector; - -template -using EigenScalar = framework::EigenScalar; - template class AccuracyKernel : public framework::OpKernel { public: diff --git a/paddle/operators/array_operator.h b/paddle/operators/array_operator.h new file mode 100644 index 0000000000000..666043e824f88 --- /dev/null +++ b/paddle/operators/array_operator.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { +class ArrayOp : public framework::OperatorBase { + public: + ArrayOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + protected: + size_t GetOffset(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const { + auto *i = scope.FindVar(Input("I")); + PADDLE_ENFORCE(i != nullptr, "I must be set"); + auto &i_tensor = i->Get(); + PADDLE_ENFORCE_EQ(i_tensor.numel(), 1); + size_t offset; + if (platform::is_gpu_place(i_tensor.place())) { + // FIXME: Avoid copy from GPU to CPU + framework::Tensor t; + t.CopyFrom(i_tensor, platform::CPUPlace(), dev_ctx); + dev_ctx.Wait(); + offset = static_cast(*t.data()); + } else { + offset = static_cast(*i_tensor.data()); + } + return offset; + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/array_to_lod_tensor_op.cc b/paddle/operators/array_to_lod_tensor_op.cc index 6cd9c06b8ae3d..c0903bb4e5ca7 100644 --- a/paddle/operators/array_to_lod_tensor_op.cc +++ b/paddle/operators/array_to_lod_tensor_op.cc @@ -140,6 +140,23 @@ class ArrayToLoDTensorInferShape : public framework::InferShapeBase { "ArrayToLoDTensorOp must has input X."); PADDLE_ENFORCE(context->HasInput("RankTable"), "ArrayToLoDTensorOp must has input RankTable."); + context->SetOutputDim("Out", context->GetInputDim("X")); + } +}; + +class ArrayToLoDTensorGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("lod_tensor_to_array"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetInput("RankTable", Input("RankTable")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); } }; @@ -149,4 +166,5 @@ class ArrayToLoDTensorInferShape : public framework::InferShapeBase { namespace ops = paddle::operators; REGISTER_OPERATOR(array_to_lod_tensor, ops::ArrayToLoDTensorOp, ops::ArrayToLoDTensorOpProtoMaker, - ops::ArrayToLoDTensorInferShape); + ops::ArrayToLoDTensorInferShape, + ops::ArrayToLoDTensorGradMaker); diff --git a/paddle/operators/assign_op.cc b/paddle/operators/assign_op.cc new file mode 100644 index 0000000000000..609e915b932e2 --- /dev/null +++ b/paddle/operators/assign_op.cc @@ -0,0 +1,138 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/data_type.h" +#include "paddle/framework/op_registry.h" +#include "paddle/framework/var_type.h" + +namespace paddle { +namespace operators { +class AssignFunctor { + public: + AssignFunctor(framework::Variable *out, + const platform::DeviceContext &dev_ctx) + : out_(out), dev_ctx_(dev_ctx) {} + + void operator()(const framework::LoDTensor &lod_tensor) const { + auto &out_tensor = *out_->GetMutable(); + copy_tensor(lod_tensor, &out_tensor); + } + + void operator()(const framework::LoDTensorArray &array) const { + auto &out_array = *out_->GetMutable(); + out_array.resize(array.size()); + for (size_t i = 0; i < array.size(); ++i) { + copy_tensor(array[i], &out_array[i]); + } + } + + void operator()(const framework::SelectedRows &rows) const { + framework::SelectedRows &out_rows = + *out_->GetMutable(); + out_rows.set_rows(rows.rows()); + out_rows.set_height(rows.height()); + auto &t = rows.value(); + out_rows.mutable_value()->CopyFrom(t, t.place(), dev_ctx_); + } + + template + void operator()(const T &v) const { + PADDLE_THROW("Not support type for assign op %s", typeid(T).name()); + } + + private: + void copy_tensor(const framework::LoDTensor &lod_tensor, + framework::LoDTensor *out) const { + auto &out_tensor = *out; + out_tensor.CopyFrom(lod_tensor, lod_tensor.place(), dev_ctx_); + out_tensor.set_lod(lod_tensor.lod()); + } + + framework::Variable *out_; + const platform::DeviceContext &dev_ctx_; +}; + +class AssignOp : public framework::OperatorBase { + public: + AssignOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *x = scope.FindVar(Input("X")); + if (x == nullptr) { + return; + } + auto *out = scope.FindVar(Output("Out")); + PADDLE_ENFORCE( + out != nullptr, + "The Output(Out) should not be null if the Input(X) is set."); + framework::VisitVarType(*x, AssignFunctor(out, dev_ctx)); + } +}; + +class AssignOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + AssignOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(LoDTensor, SelectedRows or LoDTensorArray) The input variable " + "could be LoDTensor, SelectedRows or LoDTensorArray.") + .AsDispensable(); + AddOutput("Out", + "(LoDTensor, SelectedRows or LoDTensorArray) The type of output " + "is the same as input X."); + AddComment(R"DOC(Assign Operator + +Out = X, when type in [LoDTensor/SelectedRows/LoDTensorArray] +raise error if the type is not listed above. +)DOC"); + } +}; + +class AssignInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + if (context->HasInput("X")) { + auto type = context->GetInputsVarType("X")[0]; + if (type == framework::VarDesc_VarType_SELECTED_ROWS || + type == framework::VarDesc_VarType_LOD_TENSOR) { + context->SetOutputDim("Out", context->GetInputDim("X")); + } + } + } +}; + +class AssignGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *op = new framework::OpDescBind(); + op->SetType("assign"); + op->SetInput("X", OutputGrad("Out")); + op->SetOutput("Out", InputGrad("X")); + return std::unique_ptr(op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(assign, ops::AssignOp, ops::AssignGradMaker, + ops::AssignInferShape, ops::AssignOpProtoMaker); diff --git a/paddle/operators/batch_norm_op.cc b/paddle/operators/batch_norm_op.cc index 8721ca352848f..f884e6efa917c 100644 --- a/paddle/operators/batch_norm_op.cc +++ b/paddle/operators/batch_norm_op.cc @@ -19,9 +19,6 @@ namespace operators { using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -template -using EigenMatrix = framework::EigenMatrix; template using EigenArrayMap = diff --git a/paddle/operators/beam_search_decode_op.cc b/paddle/operators/beam_search_decode_op.cc new file mode 100644 index 0000000000000..1ba4dfcdaba49 --- /dev/null +++ b/paddle/operators/beam_search_decode_op.cc @@ -0,0 +1,110 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/beam_search_decode_op.h" + +namespace paddle { +namespace operators { + +class BeamSearchDecodeOp : public framework::OperatorBase { + public: + BeamSearchDecodeOp(const std::string& type, + const framework::VariableNameMap& inputs, + const framework::VariableNameMap& outputs, + const framework::AttributeMap& attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope& scope, + const platform::DeviceContext& dev_ctx) const override { + framework::ExecutionContext ctx(*this, scope, dev_ctx); + const LoDTensorArray* ids = ctx.Input("Ids"); + const LoDTensorArray* scores = ctx.Input("Scores"); + const size_t step_num = ids->size(); + PADDLE_ENFORCE_GT(step_num, 0UL, + "beam search steps should be larger than 0"); + const size_t source_num = ids->at(0).lod().at(0).size() - 1; + PADDLE_ENFORCE_GT(source_num, 0UL, "source num should be larger than 0"); + + for (size_t i = 0; i < step_num; ++i) { + PADDLE_ENFORCE_EQ(ids->at(i).lod().size(), 2UL, + "Level of LodTensor should be 2"); + } + + // prepare output + LoDTensor* sentenceIds = ctx.Output("SentenceIds"); + LoDTensor* sentenceScores = ctx.Output("SentenceScores"); + + BeamSearchDecoder beam_search_decoder; + beam_search_decoder.PackAllSteps(*ids, *scores, sentenceIds, + sentenceScores); + } +}; + +class BeamSearchDecodeOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + BeamSearchDecodeOpProtoMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Ids", + "(LodTensorArray)" + "score of the candidate words in each step"); + AddInput("Scores", + "(LodTensorArray)" + "score of the candidate words in each step"); + AddOutput("SentenceIds", + "(LodTensor)" + "All possible result sentences of word ids"); + AddOutput("SentenceScores", + "(LodTensor)" + "All possible result sentences of word scores"); + AddComment(R"DOC( +Pack the result of Beam search op into SentenceIds and SentenceScores. +)DOC"); + } +}; + +class BeamSearchDecodeInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext* context) const override { + PADDLE_ENFORCE(context->HasInput("Ids"), + "BeamSearchDecodeOp must has input Ids"); + PADDLE_ENFORCE(context->HasInput("Scores"), + "BeamSearchDecodeOp must has input Scores"); + PADDLE_ENFORCE(context->HasOutput("SentenceIds"), + "BeamSearchDecodeOp must has output SentenceIds"); + PADDLE_ENFORCE(context->HasOutput("SentenceScores"), + "BeamSearchDecodeOp must has output SentenceScores"); + } +}; + +class BeamSearchDecodeInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDescBind& op_desc, + framework::BlockDescBind* block) const override { + for (auto& o : op_desc.Output("SentenceIds")) { + block->Var(o)->SetType(framework::VarDesc::LOD_TENSOR); + } + for (auto& o : op_desc.Output("SentenceScores")) { + block->Var(o)->SetType(framework::VarDesc::LOD_TENSOR); + } + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(beam_search_decode, paddle::operators::BeamSearchDecodeOp, + paddle::operators::BeamSearchDecodeOpProtoMaker, + paddle::operators::BeamSearchDecodeInferShape, + paddle::operators::BeamSearchDecodeInferVarType, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/beam_search_decode_op.h b/paddle/operators/beam_search_decode_op.h new file mode 100644 index 0000000000000..0f007ec22f9a6 --- /dev/null +++ b/paddle/operators/beam_search_decode_op.h @@ -0,0 +1,280 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using LoDTensor = framework::LoDTensor; +using LoDTensorArray = framework::LoDTensorArray; + +// all the lod have 2 levels. +// The First is source level, the second is sentence level. +// source level describe how many candidate words for this source. +// sentence level describe these candidates belong to which prefix +const size_t kSourceLevel = 0; +const size_t kSentenceLevel = 1; + +template +struct BeamNode { + BeamNode(int64_t word_id, T score) : word_id_(word_id), score_(score) {} + + ~BeamNode() { + if (parent_) { + parent_->DropKid(this); + if (parent_->kids_.size() == 0UL) { + delete parent_; + } + } + VLOG(3) << "Delete BeamNode root with word_id:" << this->word_id_; + } + + void AppendTo(BeamNode* parent) { + parent_ = parent; + parent->kids_.insert(this); + } + + void DropKid(BeamNode* kid) { kids_.erase(kid); } + + BeamNode* parent_ = nullptr; + std::unordered_set kids_; + int64_t word_id_; + T score_; +}; + +template +using BeamNodeVector = std::vector>>; + +template +struct Sentence { + std::vector word_ids; + std::vector scores; +}; + +template +using SentenceVector = std::vector>; + +template +struct BeamSearchDecoder { + /** + * make a BeamNode and all it's related prefix BeanNode into a Sentence. + */ + Sentence MakeSentence(const BeamNode* node) const; + + /** + * Param: + * cur_ids: LoDTensor of One step for word ID + * cur_scores: LoDTensor of One Step for word score + * prefixes_list: prefixes for each source sentence. + * sentence_vector_list: result sentence_vector for each source sentence. + * Return: + * a new prefixes list for each source of current step + */ + std::vector> PackTwoSteps( + const LoDTensor& cur_ids, const LoDTensor& cur_scores, + std::vector>& prefixes_list, + std::vector>* sentence_vector_list) const; + + /** + * convert the result sentence_vector for each source sentence into two + * LodTensor. + * One is all candidate sentences with word id, one is all candidate sentences + * with word score. + * Param: + * sentence_vector_list: sentence_vector for each source sentence. + * id_tensor: result LoDTensor for sentences of id. + * score_tensor: result LoDTensor for sentences of score. + */ + void ConvertSentenceVectorToLodTensor( + std::vector> sentence_vector_list, LoDTensor* id_tensor, + LoDTensor* score_tensor) const; + + /** + * Pack all steps of id/score LodTensor into sentence LoDTensor + * it's main logic is: + * ```python + * prefix + * result_sentence + * result_lod_tensor + * + * for (step in steps): + * prefix = PackTwoSteps(prefix, step, &result_sentence) + * ConvertSentenceVectorToLodTensor(result_sentence, &result_lod_tensor) + * ``` + */ + void PackAllSteps(const LoDTensorArray& step_ids, + const LoDTensorArray& step_scores, LoDTensor* id_tensor, + LoDTensor* score_tensor) const; +}; + +template +Sentence BeamSearchDecoder::MakeSentence(const BeamNode* node) const { + Sentence sentence; + while (node != nullptr) { + sentence.word_ids.emplace_back(node->word_id_); + sentence.scores.emplace_back(node->score_); + node = node->parent_; + } + + std::reverse(std::begin(sentence.word_ids), std::end(sentence.word_ids)); + std::reverse(std::begin(sentence.scores), std::end(sentence.scores)); + + return sentence; +} + +template +std::vector> BeamSearchDecoder::PackTwoSteps( + const LoDTensor& cur_ids, const LoDTensor& cur_scores, + std::vector>& prefixes_list, + std::vector>* sentence_vector_list) const { + std::vector> result; + + for (size_t src_idx = 0; src_idx < cur_ids.lod()[kSourceLevel].size() - 1; + ++src_idx) { + size_t src_start = cur_ids.lod().at(kSourceLevel)[src_idx]; + size_t src_end = cur_ids.lod().at(kSourceLevel)[src_idx + 1]; + + BeamNodeVector beam_nodes; + + // if prefixes size is 0, it means this is the first step. In this step, + // all candidate id is the start of candidate sentences. + if (prefixes_list.empty()) { + PADDLE_ENFORCE_EQ(cur_ids.lod().at(kSourceLevel).back(), + cur_ids.lod().at(kSentenceLevel).back(), + "in the first step"); + for (size_t id_idx = src_start; id_idx < src_end; ++id_idx) { + beam_nodes.push_back(std::unique_ptr>(new BeamNode( + cur_ids.data()[id_idx], cur_scores.data()[id_idx]))); + } + } else { + BeamNodeVector& prefixes = prefixes_list[src_idx]; + SentenceVector& sentence_vector = (*sentence_vector_list)[src_idx]; + + PADDLE_ENFORCE_EQ(src_end - src_start, prefixes.size(), + "prefix and candidate set number should be the same"); + + auto candidate_offset = cur_ids.lod()[kSentenceLevel]; + for (size_t prefix_idx = 0; prefix_idx < prefixes.size(); ++prefix_idx) { + std::unique_ptr>& prefix = prefixes[prefix_idx]; + size_t candidate_start = candidate_offset[src_start + prefix_idx]; + size_t candidate_end = candidate_offset[src_start + prefix_idx + 1]; + if (candidate_start == candidate_end) { + VLOG(3) << "this sentence has no more candidate, " + "add to result sentence and rm it from beam tree"; + sentence_vector.push_back(MakeSentence(prefix.get())); + prefix.reset(); + } else { + for (size_t candidate_idx = candidate_start; + candidate_idx < candidate_end; ++candidate_idx) { + auto* candidate = + new BeamNode(cur_ids.data()[candidate_idx], + cur_scores.data()[candidate_idx]); + candidate->AppendTo(prefix.get()); + beam_nodes.push_back(std::unique_ptr>(candidate)); + } + prefix.release(); + } + } + } + result.push_back(std::move(beam_nodes)); + } + return result; +} + +template +void BeamSearchDecoder::ConvertSentenceVectorToLodTensor( + std::vector> sentence_vector_list, LoDTensor* id_tensor, + LoDTensor* score_tensor) const { + size_t src_num = sentence_vector_list.size(); + + PADDLE_ENFORCE_NE(src_num, 0, "src_num should not be 0"); + + std::vector source_level_lod = {0}; + std::vector sentence_level_lod = {0}; + std::vector id_data; + std::vector score_data; + + for (size_t src_idx = 0; src_idx < src_num; ++src_idx) { + for (Sentence& sentence : sentence_vector_list[src_idx]) { + id_data.insert(id_data.end(), sentence.word_ids.begin(), + sentence.word_ids.end()); + score_data.insert(score_data.end(), sentence.scores.begin(), + sentence.scores.end()); + sentence_level_lod.push_back(sentence_level_lod.back() + + sentence.word_ids.size()); + } + source_level_lod.push_back(source_level_lod.back() + + sentence_vector_list[src_idx].size()); + } + + auto cpu_place = new paddle::platform::CPUPlace(); + paddle::platform::CPUDeviceContext cpu_ctx(*cpu_place); + + framework::LoD lod; + lod.push_back(source_level_lod); + lod.push_back(sentence_level_lod); + + id_tensor->set_lod(lod); + id_tensor->Resize({static_cast(id_data.size())}); + id_tensor->mutable_data(paddle::platform::CPUPlace()); + id_tensor->CopyFromVector(id_data, cpu_ctx); + + score_tensor->set_lod(lod); + score_tensor->Resize({static_cast(score_data.size())}); + score_tensor->mutable_data(paddle::platform::CPUPlace()); + score_tensor->CopyFromVector(score_data, cpu_ctx); +} + +template +void BeamSearchDecoder::PackAllSteps(const LoDTensorArray& step_ids, + const LoDTensorArray& step_scores, + LoDTensor* id_tensor, + LoDTensor* score_tensor) const { + PADDLE_ENFORCE(!step_ids.empty(), "step num should be larger than 0"); + PADDLE_ENFORCE_EQ(step_ids.size(), step_scores.size(), + "step_ids and step_scores should be the same"); + const size_t step_num = step_ids.size(); + const size_t src_num = step_ids.at(0).lod().at(kSourceLevel).size() - 1; + + PADDLE_ENFORCE_GT(src_num, 0UL, "source num should be larger than 0"); + + // previous prefixes for each step, + // the init length is 0, means this is the first step. + std::vector> beamnode_vector_list(0); + std::vector> sentence_vector_list(src_num); + + // pack all steps for one batch first, then another batch + for (size_t step_id = 0; step_id < step_num; ++step_id) { + beamnode_vector_list = + PackTwoSteps(step_ids.at(step_id), step_scores.at(step_id), + beamnode_vector_list, &sentence_vector_list); + } + // append last beam_node to result + for (size_t src_idx = 0; src_idx < src_num; ++src_idx) { + for (auto& beam_node : beamnode_vector_list.at(src_idx)) { + sentence_vector_list[src_idx].push_back(MakeSentence(beam_node.get())); + beam_node.reset(); + } + } + + ConvertSentenceVectorToLodTensor(sentence_vector_list, id_tensor, + score_tensor); +} + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/beam_search_decode_op_test.cc b/paddle/operators/beam_search_decode_op_test.cc new file mode 100644 index 0000000000000..5ac23991f3c77 --- /dev/null +++ b/paddle/operators/beam_search_decode_op_test.cc @@ -0,0 +1,221 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/beam_search_decode_op.h" +#include "gtest/gtest.h" + +using CPUPlace = paddle::platform::CPUPlace; +using LoD = paddle::framework::LoD; +using LoDTensor = paddle::framework::LoDTensor; +using LoDTensorArray = paddle::framework::LoDTensorArray; + +template +using BeamNode = paddle::operators::BeamNode; +template +using BeamSearchDecoder = paddle::operators::BeamSearchDecoder; +template +using Sentence = paddle::operators::Sentence; +template +using BeamNodeVector = paddle::operators::BeamNodeVector; +template +using SentenceVector = paddle::operators::SentenceVector; + +namespace paddle { +namespace test { + +void GenerateExample(const std::vector& level_0, + const std::vector& level_1, + const std::vector& data, LoDTensorArray* ids, + LoDTensorArray* scores) { + PADDLE_ENFORCE_EQ(level_0.back(), level_1.size() - 1, + "source level is used to describe candidate set"); + PADDLE_ENFORCE_EQ(level_1.back(), data.size(), + "the lowest level is used to describe data" + ", so it's last element should be data length"); + + CPUPlace place; + + LoD lod; + lod.push_back(level_0); + lod.push_back(level_1); + + // Ids + LoDTensor tensor_id; + tensor_id.set_lod(lod); + tensor_id.Resize({static_cast(data.size())}); + // malloc memory + int64_t* id_ptr = tensor_id.mutable_data(place); + for (size_t i = 0; i < data.size(); ++i) { + id_ptr[i] = static_cast(data.at(i)); + } + + // Scores + LoDTensor tensor_score; + tensor_score.set_lod(lod); + tensor_score.Resize({static_cast(data.size())}); + // malloc memory + float* score_ptr = tensor_score.mutable_data(place); + for (size_t i = 0; i < data.size(); ++i) { + score_ptr[i] = static_cast(data.at(i)); + } + + ids->push_back(tensor_id); + scores->push_back(tensor_score); +} + +} // namespace test +} // namespace paddle + +TEST(BeamSearchDecodeOp, DeleteBeamNode) { + auto* root = new BeamNode(0, 0); + auto* b1 = new BeamNode(1, 1); + auto* b2 = new BeamNode(2, 2); + auto* b3 = new BeamNode(3, 3); + + b1->AppendTo(root); + b2->AppendTo(root); + b3->AppendTo(b1); + + delete b3; + delete b2; +} + +TEST(BeamSearchDecodeOp, MakeSentence) { + auto* root = new BeamNode(0, 0); + auto* b1 = new BeamNode(1, 1); + auto* end = new BeamNode(2, 2); + b1->AppendTo(root); + end->AppendTo(b1); + + BeamSearchDecoder helper; + Sentence sentence = helper.MakeSentence(end); + delete end; + + std::vector expect_ids = {0, 1, 2}; + ASSERT_EQ(sentence.word_ids, expect_ids); + + std::vector expect_scores = {0, 1, 2}; + ASSERT_EQ(sentence.scores, expect_scores); +} + +TEST(BeamSearchDecodeOp, PackTwoStepsFistStep) { + CPUPlace place; + + LoDTensorArray ids; + LoDTensorArray scores; + + paddle::test::GenerateExample( + std::vector{0, 2, 6}, std::vector{0, 1, 2, 3, 4, 5, 6}, + std::vector{1, 2, 3, 4, 5, 6}, &ids, &scores); + + std::vector> beamnode_vector_list; + std::vector> sentence_vector_list( + 2, SentenceVector()); + + BeamSearchDecoder helper; + beamnode_vector_list = helper.PackTwoSteps( + ids[0], scores[0], beamnode_vector_list, &sentence_vector_list); + ASSERT_EQ(beamnode_vector_list.size(), 2UL); + ASSERT_EQ(beamnode_vector_list[0].size(), 2UL); + ASSERT_EQ(beamnode_vector_list[1].size(), 4UL); +} + +TEST(BeamSearchDecodeOp, PackTwoSteps) { + CPUPlace place; + + // first source has three prefix + BeamNodeVector source0_prefixes; + source0_prefixes.push_back( + std::unique_ptr>(new BeamNode(1, 1))); + source0_prefixes.push_back( + std::unique_ptr>(new BeamNode(0, 0))); + source0_prefixes.push_back( + std::unique_ptr>(new BeamNode(3, 3))); + + // second source has two prefix + BeamNodeVector source1_prefixes; + source1_prefixes.push_back( + std::unique_ptr>(new BeamNode(4, 4))); + source1_prefixes.push_back( + std::unique_ptr>(new BeamNode(5, 5))); + + std::vector> beamnode_vector_list; + std::vector> sentence_vector_list( + 2, SentenceVector()); + + beamnode_vector_list.push_back(std::move(source0_prefixes)); + beamnode_vector_list.push_back(std::move(source1_prefixes)); + + // generate data for one step + LoDTensorArray ids; + LoDTensorArray scores; + + paddle::test::GenerateExample(std::vector{0, 3, 5}, + std::vector{0, 1, 1, 3, 4, 5}, + std::vector{0, 1, 2, 3, 4}, &ids, &scores); + + BeamSearchDecoder helper1; + beamnode_vector_list = helper1.PackTwoSteps( + ids[0], scores[0], beamnode_vector_list, &sentence_vector_list); + + ASSERT_EQ(sentence_vector_list[0].size(), 1UL); + ASSERT_EQ(sentence_vector_list[1].size(), 0UL); + ASSERT_EQ(beamnode_vector_list[0].size(), 3UL); + ASSERT_EQ(beamnode_vector_list[1].size(), 2UL); +} + +TEST(BeamSearchDecodeOp, PackAllSteps) { + CPUPlace place; + + // we will constuct a sample data with 3 steps and 2 source sentences + LoDTensorArray ids; + LoDTensorArray scores; + + paddle::test::GenerateExample( + std::vector{0, 3, 6}, std::vector{0, 1, 2, 3, 4, 5, 6}, + std::vector{1, 2, 3, 4, 5, 6}, &ids, &scores); + paddle::test::GenerateExample( + std::vector{0, 3, 6}, std::vector{0, 1, 1, 3, 5, 5, 6}, + std::vector{0, 1, 2, 3, 4, 5}, &ids, &scores); + paddle::test::GenerateExample(std::vector{0, 3, 6}, + std::vector{0, 0, 1, 2, 3, 4, 5}, + std::vector{0, 1, 2, 3, 4}, &ids, &scores); + + ASSERT_EQ(ids.size(), 3UL); + ASSERT_EQ(scores.size(), 3UL); + + BeamSearchDecoder helper; + + LoDTensor id_tensor; + LoDTensor score_tensor; + helper.PackAllSteps(ids, scores, &id_tensor, &score_tensor); + + LoD lod = id_tensor.lod(); + std::vector expect_source_lod = {0, 4, 8}; + EXPECT_EQ(lod[0], expect_source_lod); + std::vector expect_sentence_lod = {0, 1, 3, 6, 9, 10, 13, 16, 19}; + EXPECT_EQ(lod[1], expect_sentence_lod); + // 2| 1, 0| 3, 1, 0| 3, 2, 1| 5| 4, 3, 2| 4, 4, 3| 6, 5, 4 + std::vector expect_data = {2, 1, 0, 3, 1, 0, 3, 2, 1, 5, + 4, 3, 2, 4, 4, 3, 6, 5, 4}; + ASSERT_EQ(id_tensor.dims()[0], static_cast(expect_data.size())); + for (size_t i = 0; i < expect_data.size(); ++i) { + ASSERT_EQ(id_tensor.data()[i], + static_cast(expect_data[i])); + } + for (int64_t i = 0; i < id_tensor.dims()[0]; ++i) { + ASSERT_EQ(score_tensor.data()[i], + static_cast(id_tensor.data()[i])); + } +} diff --git a/paddle/operators/bilinear_tensor_product_op.cc b/paddle/operators/bilinear_tensor_product_op.cc new file mode 100644 index 0000000000000..c65ba7eb262f3 --- /dev/null +++ b/paddle/operators/bilinear_tensor_product_op.cc @@ -0,0 +1,159 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/bilinear_tensor_product_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class BilinearTensorProductOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null."); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto weight_dims = ctx->GetInputDim("Weight"); + + PADDLE_ENFORCE_EQ(x_dims.size(), 2UL, "The input(X) must be a 2D Tensor."); + PADDLE_ENFORCE_EQ(y_dims.size(), 2UL, "The input(Y) must be a 2D Tensor."); + PADDLE_ENFORCE_EQ(weight_dims.size(), 3UL, + "The input(Weight) must be a 3D tensor."); + PADDLE_ENFORCE_EQ(x_dims[0], y_dims[0], + "The first dimension(batch_size) of input(X) must be " + "equal to the first dimension of the input(Y)."); + PADDLE_ENFORCE_EQ(x_dims[1], weight_dims[1], + "The second dimension of input(X) must be equal to " + "the second dimension of the input(Weight)."); + PADDLE_ENFORCE_EQ(y_dims[1], weight_dims[2], + "The second dimension of input(Y) must be equal to " + "the third dimension of the input(Weight)."); + + if (ctx->HasInput("Bias")) { + auto bias_dims = ctx->GetInputDim("Bias"); + PADDLE_ENFORCE(bias_dims.size() == 2UL && bias_dims[0] == 1UL, + "The Input(Bias) must be a 2-D tensor with " + "the 2nd dimension fixed to 1 (a row vector)."); + PADDLE_ENFORCE_EQ(bias_dims[1], weight_dims[0], + "The second dimension of input(Bias) must be equal " + "to the first dimension of the input(Weight)."); + } + + ctx->SetOutputDim("Out", {x_dims[0], weight_dims[0]}); + ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +class BilinearTensorProductOpMaker : public framework::OpProtoAndCheckerMaker { + public: + BilinearTensorProductOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The first input of bilinear_tensor_product operator."); + AddInput("Y", "The second input of bilinear_tensor_product operator."); + AddInput("Weight", + "The learnable parameters of bilinear_tensor_product operator."); + AddInput("Bias", "The learnable bias of bilinear_tensor_product operator.") + .AsDispensable(); + AddOutput("Out", "The output of bilinear_tensor_product operator."); + AddComment(R"DOC( +Bilinear Tensor Product operator. +Given input X and Y, a 3D tensor weight, and bias. Each column of the +output is computed by one slice i = 1, . . . , k of the tensor: + + M = (X W_i) \cdot Y + Out_i = \sum_i {M_i} + Bias_i + +)DOC"); + } +}; + +class BilinearTensorProductOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) should not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null."); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto weight_dims = ctx->GetInputDim("Weight"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); + + PADDLE_ENFORCE_EQ(out_dims.size(), 2UL, + "The input(Out@GRAD) must be a 2D Tensor."); + PADDLE_ENFORCE_EQ( + x_dims[0], out_dims[0], + "The first dimension(batch_size) of input(Out@GRAD) must be " + "equal to the first dimension of the Input(X)."); + PADDLE_ENFORCE_EQ( + weight_dims[0], out_dims[1], + "The second dimension of input(Out@GRAD) must be equal to " + "the third dimension of the Input(Weight)."); + + if (ctx->HasInput("Bias")) { + auto bias_dims = ctx->GetInputDim("Bias"); + PADDLE_ENFORCE_EQ( + bias_dims[1], out_dims[1], + "The second dimension of input(Out@GRAD) must be equal to " + "the second dimension of the Input(Bias)."); + auto bias_grad_name = framework::GradVarName("Bias"); + if (ctx->HasOutput(bias_grad_name)) + ctx->SetOutputDim(bias_grad_name, bias_dims); + } + + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + auto weight_grad_name = framework::GradVarName("Weight"); + + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + if (ctx->HasOutput(y_grad_name)) { + ctx->SetOutputDim(y_grad_name, y_dims); + } + if (ctx->HasOutput(weight_grad_name)) { + ctx->SetOutputDim(weight_grad_name, weight_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(bilinear_tensor_product, ops::BilinearTensorProductOp, + ops::BilinearTensorProductOpMaker, bilinear_tensor_product_grad, + ops::BilinearTensorProductOpGrad); +REGISTER_OP_CPU_KERNEL( + bilinear_tensor_product, + ops::BilinearTensorProductKernel, + ops::BilinearTensorProductKernel); +REGISTER_OP_CPU_KERNEL( + bilinear_tensor_product_grad, + ops::BilinearTensorProductGradKernel, + ops::BilinearTensorProductGradKernel); diff --git a/paddle/operators/bilinear_tensor_product_op.cu b/paddle/operators/bilinear_tensor_product_op.cu new file mode 100644 index 0000000000000..858d2668d0137 --- /dev/null +++ b/paddle/operators/bilinear_tensor_product_op.cu @@ -0,0 +1,26 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/bilinear_tensor_product_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + bilinear_tensor_product, + ops::BilinearTensorProductKernel, + ops::BilinearTensorProductKernel); +REGISTER_OP_GPU_KERNEL( + bilinear_tensor_product_grad, + ops::BilinearTensorProductGradKernel, + ops::BilinearTensorProductGradKernel); diff --git a/paddle/operators/bilinear_tensor_product_op.h b/paddle/operators/bilinear_tensor_product_op.h new file mode 100644 index 0000000000000..ffa4f43a32741 --- /dev/null +++ b/paddle/operators/bilinear_tensor_product_op.h @@ -0,0 +1,184 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +template +using EigenMatrix = framework::EigenMatrix; + +template +class BilinearTensorProductKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* weight = ctx.Input("Weight"); + auto* bias = ctx.Input("Bias"); + auto* out = ctx.Output("Out"); + out->mutable_data(ctx.GetPlace()); + + auto y_mat = EigenMatrix::From(*y); + auto output_mat = EigenMatrix::From(*out); + + auto batch_size = x->dims()[0]; + auto weight_dims = weight->dims(); + int out_dim = weight_dims[0]; + auto x_dim = weight_dims[1]; + auto y_dim = weight_dims[2]; + auto place = ctx.GetEigenDevice(); + + // Create the intermediate variable to caculate the result of + // Input(X) multiplied by Input(Weight_i), the formula is: + // left_mul = X Weight_i. + Tensor left_mul; + left_mul.mutable_data(framework::make_ddim({batch_size, y_dim}), + ctx.GetPlace()); + auto left_mul_mat = EigenMatrix::From(left_mul); + + for (int i = 0; i < out_dim; ++i) { + auto output_col_vec = output_mat.chip(i, 1); + Tensor weight_mat = + weight->Slice(i, i + 1).Resize(framework::make_ddim({x_dim, y_dim})); + math::gemm(ctx.device_context(), CblasNoTrans, CblasNoTrans, + batch_size, y_dim, x_dim, 1, x->data(), + weight_mat.data(), 0, left_mul.data()); + output_col_vec.device(place) = + (left_mul_mat * y_mat).sum(Eigen::DSizes(1)); + } + if (bias) { + auto bias_vec = EigenMatrix::From(*bias); + Eigen::DSizes bcast(batch_size, 1); + output_mat.device(place) = bias_vec.broadcast(bcast) + output_mat; + } + } +}; + +template +class BilinearTensorProductGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + const Tensor* x = ctx.Input("X"); + const Tensor* y = ctx.Input("Y"); + const Tensor* weight = ctx.Input("Weight"); + Tensor* d_x = ctx.Output(framework::GradVarName("X")); + Tensor* d_y = ctx.Output(framework::GradVarName("Y")); + Tensor* d_weight = ctx.Output(framework::GradVarName("Weight")); + Tensor* d_bias = ctx.Output(framework::GradVarName("Bias")); + const Tensor* d_out = ctx.Input(framework::GradVarName("Out")); + + auto batch_size = x->dims()[0]; + auto weight_dims = weight->dims(); + int out_dim = weight_dims[0]; + auto x_dim = weight_dims[1]; + auto y_dim = weight_dims[2]; + + auto x_mat = EigenMatrix::From(*x); + auto y_mat = EigenMatrix::From(*y); + auto d_out_mat = EigenMatrix::From(*d_out); + auto place = ctx.GetEigenDevice(); + + // Create the intermediate variable to caculate the Output(Y@Grad). + Tensor x_scale; + x_scale.mutable_data(framework::make_ddim({batch_size, x_dim}), + ctx.GetPlace()); + auto x_scale_mat = EigenMatrix::From(x_scale); + + // Create the intermediate variable to caculate the Output(X@Grad). + Tensor y_scale; + y_scale.mutable_data(framework::make_ddim({batch_size, y_dim}), + ctx.GetPlace()); + auto y_scale_mat = EigenMatrix::From(y_scale); + + math::SetConstant set_zero; + + // Set Output(X@Grad) be zero. + if (d_x) { + d_x->mutable_data(ctx.GetPlace()); + set_zero(ctx.device_context(), d_x, static_cast(0)); + } + + // Set Output(Y@Grad) be zero. + if (d_y) { + d_y->mutable_data(ctx.GetPlace()); + set_zero(ctx.device_context(), d_y, static_cast(0)); + } + + // Caculate the Output(X@Grad) and Output(Y@Grad). + if (d_x || d_y) { + Eigen::DSizes bcast_for_x(1, y_dim); + Eigen::DSizes bcast_for_y(1, x_dim); + for (int i = 0; i < out_dim; ++i) { + Tensor weight_i = weight->Slice(i, i + 1).Resize( + framework::make_ddim({x_dim, y_dim})); + auto output_vec = d_out_mat.chip(i, 1); + if (d_x) { + y_scale_mat.device(place) = + output_vec.reshape(Eigen::DSizes(batch_size, 1)) + .broadcast(bcast_for_x) * + y_mat; + math::gemm(ctx.device_context(), CblasNoTrans, CblasTrans, + batch_size, x_dim, y_dim, 1, y_scale.data(), + weight_i.data(), 1, d_x->data()); + } + if (d_y) { + x_scale_mat.device(place) = + output_vec.reshape(Eigen::DSizes(batch_size, 1)) + .broadcast(bcast_for_y) * + x_mat; + math::gemm(ctx.device_context(), CblasNoTrans, CblasNoTrans, + batch_size, y_dim, x_dim, 1, x_scale.data(), + weight_i.data(), 1, d_y->data()); + } + } + } + + // Caculate the gradient of Input(Weight). + if (d_weight) { + d_weight->mutable_data(ctx.GetPlace()); + Eigen::DSizes bcast_for_weight(1, x_dim); + for (int i = 0; i < out_dim; ++i) { + Tensor d_weight_i = d_weight->Slice(i, i + 1).Resize( + framework::make_ddim({x_dim, y_dim})); + auto output_vec = d_out_mat.chip(i, 1); + x_scale_mat.device(place) = + output_vec.reshape(Eigen::DSizes(batch_size, 1)) + .broadcast(bcast_for_weight) * + x_mat; + math::gemm(ctx.device_context(), CblasTrans, CblasNoTrans, + x_dim, y_dim, batch_size, 1, x_scale.data(), + y->data(), 0, d_weight_i.data()); + } + } + + // Caculate the gradient of Input(Bias). + if (d_bias) { + d_bias->mutable_data(ctx.GetPlace()); + auto d_bias_mat = EigenMatrix::From(*d_bias); + d_bias_mat.device(place) = d_out_mat.sum(Eigen::DSizes(0)); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/chunk_eval_op.cc b/paddle/operators/chunk_eval_op.cc new file mode 100644 index 0000000000000..309660b01fe70 --- /dev/null +++ b/paddle/operators/chunk_eval_op.cc @@ -0,0 +1,145 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/chunk_eval_op.h" + +namespace paddle { +namespace operators { + +class ChunkEvalOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Inference"), + "Input(Inference) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), + "Input(Label) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Precision"), + "Output(Precision) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Recall"), + "Output(Recall) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("F1-Score"), + "Output(F1-Score) of ChunkEvalOp should not be null."); + + auto inference_dim = ctx->GetInputDim("Inference"); + auto label_dim = ctx->GetInputDim("Label"); + + PADDLE_ENFORCE(inference_dim == label_dim, + "Inference's shape must be the same as Label's shape."); + + ctx->SetOutputDim("Precision", {1}); + ctx->SetOutputDim("Recall", {1}); + ctx->SetOutputDim("F1-Score", {1}); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType(framework::DataType::FP32, + ctx.device_context()); + } +}; + +class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ChunkEvalOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Inference", + "(Tensor, default: Tensor). Predictions from the network."); + AddInput("Label", + "(Tensor, default: Tensor). The true tag sequences."); + AddOutput("Precision", + "(float). The evaluated precision (called positive predictive " + "value) of chunks on the given mini-batch."); + AddOutput("Recall", + "(float). The evaluated recall (true positive rate or " + "sensitivity) of chunks on the given mini-batch."); + AddOutput("F1-Score", + "(float). The evaluated F1-Score on the given mini-batch."); + AddAttr("num_chunk_types", + "(int). The number of chunk type. See below for details."); + AddAttr( + "chunk_scheme", + "(string, default IOB). The labeling scheme indicating " + "how to encode the chunks. Must be IOB, IOE, IOBES or plain. See below " + "for details.") + .SetDefault("IOB"); + AddAttr>("excluded_chunk_types", + "(list) A list including chunk type ids " + "indicating chunk types that are not counted. " + "See below for details.") + .SetDefault(std::vector{}); + AddComment(R"DOC( +For some basics of chunking, please refer to +‘Chunking with Support Vector Mechines ’. + + +CheckEvalOp computes the precision, recall, and F1-score of chunk detection, +and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes. +Here is a NER example of labeling for these tagging schemes: + + Li Ming works at Agricultural Bank of China in Beijing. + IO: I-PER I-PER O O I-ORG I-ORG I-ORG I-ORG O I-LOC + IOB: B-PER I-PER O O B-ORG I-ORG I-ORG I-ORG O B-LOC + IOE: I-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O E-LOC + IOBES: B-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O S-LOC + +There are three chunk types(named entity types) including PER(person), ORG(orgnazation) +and LOC(LOCATION), and we can see that the labels have the form -. + +Since the calculations actually use label ids rather than labels, extra attention +should be paid when mapping labels to ids to make CheckEvalOp work. The key point +is that the listed equations are satisfied by ids. + + tag_type = label % num_tag_type + chunk_type = label / num_tag_type + +where `num_tag_type` is the num of tag types in the tagging scheme, `num_chunk_type` +is the num of chunk types, and `tag_type` get its value from the following table. + + Scheme Begin Inside End Single + plain 0 - - - + IOB 0 1 - - + IOE - 0 1 - + IOBES 0 1 2 3 + +Still use NER as example, assuming the tagging scheme is IOB while chunk types are ORG, +PER and LOC. To satisfy the above equations, the label map can be like this: + + B-ORG 0 + I-ORG 1 + B-PER 2 + I-PER 3 + B-LOC 4 + I-LOC 5 + O 6 + +It’s not hard to verify the equations noting that the num of chunk types +is 3 and the num of tag types in IOB scheme is 2. For example, the label +id of I-LOC is 5, the tag type id of I-LOC is 1, and the chunk type id of +I-LOC is 2, which consistent with the results from the equations. +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(chunk_eval, ops::ChunkEvalOp, + ops::ChunkEvalOpMaker); +REGISTER_OP_CPU_KERNEL(chunk_eval, + ops::ChunkEvalKernel); diff --git a/paddle/operators/chunk_eval_op.h b/paddle/operators/chunk_eval_op.h new file mode 100644 index 0000000000000..81aa07817b673 --- /dev/null +++ b/paddle/operators/chunk_eval_op.h @@ -0,0 +1,219 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; + +template +class ChunkEvalKernel : public framework::OpKernel { + public: + struct Segment { + int begin; + int end; + int type; + bool operator==(const Segment& y) const { + return begin == y.begin && end == y.end && type == y.type; + } + }; + + void GetSegments(const int* label, int length, std::vector& segments, + int num_chunk_types, int num_tag_types, int other_chunk_type, + int tag_begin, int tag_inside, int tag_end, + int tag_single) const { + segments.clear(); + segments.reserve(length); + int chunk_start = 0; + bool in_chunk = false; + int tag = -1; + int type = other_chunk_type; + for (int i = 0; i < length; ++i) { + int prev_tag = tag; + int prev_type = type; + PADDLE_ENFORCE_LE(label[i], num_chunk_types * num_tag_types); + tag = label[i] % num_tag_types; + type = label[i] / num_tag_types; + if (in_chunk && ChunkEnd(prev_tag, prev_type, tag, type, other_chunk_type, + tag_begin, tag_inside, tag_end, tag_single)) { + Segment segment{ + chunk_start, // begin + i - 1, // end + prev_type, + }; + segments.push_back(segment); + in_chunk = false; + } + if (ChunkBegin(prev_tag, prev_type, tag, type, other_chunk_type, + tag_begin, tag_inside, tag_end, tag_single)) { + chunk_start = i; + in_chunk = true; + } + } + if (in_chunk) { + Segment segment{ + chunk_start, // begin + length - 1, // end + type, + }; + segments.push_back(segment); + } + } + + bool ChunkEnd(int prev_tag, int prev_type, int tag, int type, + int other_chunk_type, int tag_begin, int tag_inside, + int tag_end, int tag_single) const { + if (prev_type == other_chunk_type) return false; + if (type == other_chunk_type) return true; + if (type != prev_type) return true; + if (prev_tag == tag_begin) return tag == tag_begin || tag == tag_single; + if (prev_tag == tag_inside) return tag == tag_begin || tag == tag_single; + if (prev_tag == tag_end) return true; + if (prev_tag == tag_single) return true; + return false; + } + + bool ChunkBegin(int prev_tag, int prev_type, int tag, int type, + int other_chunk_type, int tag_begin, int tag_inside, + int tag_end, int tag_single) const { + if (prev_type == other_chunk_type) return type != other_chunk_type; + if (type == other_chunk_type) return false; + if (type != prev_type) return true; + if (tag == tag_begin) return true; + if (tag == tag_inside) return prev_tag == tag_end || prev_tag == tag_single; + if (tag == tag_end) return prev_tag == tag_end || prev_tag == tag_single; + if (tag == tag_single) return true; + return false; + } + + void Compute(const framework::ExecutionContext& context) const override { + // initialize to parse configurations + int num_chunk_types, num_tag_types; + int other_chunk_type; + int tag_begin, tag_inside, tag_end, tag_single; + std::vector label_segments; + std::vector output_segments; + std::set excluded_chunk_types; + int64_t num_output_segments = 0; + int64_t num_label_segments = 0; + int64_t num_correct = 0; + if (context.Attr("chunk_scheme") == "IOB") { + num_tag_types = 2; + tag_begin = 0; + tag_inside = 1; + tag_end = -1; + tag_single = -1; + } else if (context.Attr("chunk_scheme") == "IOE") { + num_tag_types = 2; + tag_begin = -1; + tag_inside = 0; + tag_end = 1; + tag_single = -1; + } else if (context.Attr("chunk_scheme") == "IOBES") { + num_tag_types = 4; + tag_begin = 0; + tag_inside = 1; + tag_end = 2; + tag_single = 3; + } else if (context.Attr("chunk_scheme") == "plain") { + num_tag_types = 1; + tag_begin = -1; + tag_inside = -1; + tag_end = -1; + tag_single = -1; + } else { + PADDLE_THROW("Unknown chunk scheme."); + } + other_chunk_type = num_chunk_types = context.Attr("num_chunk_types"); + excluded_chunk_types.insert( + context.Attr>("excluded_chunk_types").begin(), + context.Attr>("excluded_chunk_types").end()); + + auto* inference = context.Input("Inference"); + auto* label = context.Input("Label"); + auto* precision = context.Output("Precision"); + auto* recall = context.Output("Recall"); + auto* f1 = context.Output("F1-Score"); + + const int* inference_data = inference->data(); + const int* label_data = label->data(); + T* precision_data = precision->mutable_data(context.GetPlace()); + T* racall_data = recall->mutable_data(context.GetPlace()); + T* f1_data = f1->mutable_data(context.GetPlace()); + + auto lod = label->lod(); + PADDLE_ENFORCE_EQ(lod.size(), 1UL, "Only support one level sequence now."); + PADDLE_ENFORCE(lod == inference->lod(), + "LoD must be same between Inference and Label."); + int num_sequences = lod[0].size() - 1; + for (int i = 0; i < num_sequences; ++i) { + int seq_length = lod[0][i + 1] - lod[0][i]; + EvalOneSeq(inference_data + lod[0][i], label_data + lod[0][i], seq_length, + output_segments, label_segments, num_output_segments, + num_label_segments, num_correct, num_chunk_types, + num_tag_types, other_chunk_type, tag_begin, tag_inside, + tag_end, tag_single, excluded_chunk_types); + } + *precision_data = !num_output_segments ? 0 : static_cast(num_correct) / + num_output_segments; + *racall_data = !num_label_segments ? 0 : static_cast(num_correct) / + num_label_segments; + *f1_data = !num_correct ? 0 : 2 * (*precision_data) * (*racall_data) / + ((*precision_data) + (*racall_data)); + } + + void EvalOneSeq(const int* output, const int* label, int length, + std::vector& output_segments, + std::vector& label_segments, + int64_t& num_output_segments, int64_t& num_label_segments, + int64_t& num_correct, int num_chunk_types, int num_tag_types, + int other_chunk_type, int tag_begin, int tag_inside, + int tag_end, int tag_single, + const std::set& excluded_chunk_types) const { + GetSegments(output, length, output_segments, num_chunk_types, num_tag_types, + other_chunk_type, tag_begin, tag_inside, tag_end, tag_single); + GetSegments(label, length, label_segments, num_chunk_types, num_tag_types, + other_chunk_type, tag_begin, tag_inside, tag_end, tag_single); + size_t i = 0, j = 0; + while (i < output_segments.size() && j < label_segments.size()) { + if (output_segments[i] == label_segments[j] && + excluded_chunk_types.count(output_segments[i].type) != 1) { + ++num_correct; + } + if (output_segments[i].end < label_segments[j].end) { + ++i; + } else if (output_segments[i].end > label_segments[j].end) { + ++j; + } else { + ++i; + ++j; + } + } + for (auto& segment : label_segments) { + if (excluded_chunk_types.count(segment.type) != 1) ++num_label_segments; + } + for (auto& segment : output_segments) { + if (excluded_chunk_types.count(segment.type) != 1) ++num_output_segments; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/clip_by_norm_op.cc b/paddle/operators/clip_by_norm_op.cc new file mode 100644 index 0000000000000..d9fc532e39500 --- /dev/null +++ b/paddle/operators/clip_by_norm_op.cc @@ -0,0 +1,70 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/clip_by_norm_op.h" + +namespace paddle { +namespace operators { + +class ClipByNormOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of ClipByNormOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of ClipByNormOp should not be null."); + auto max_norm = ctx->Attrs().Get("max_norm"); + PADDLE_ENFORCE_GT(max_norm, 0, "max_norm should be greater than 0."); + auto x_dims = ctx->GetInputDim("X"); + ctx->SetOutputDim("Out", x_dims); + ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +class ClipByNormOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ClipByNormOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(Tensor) The input of clip_by_norm op." + "The number of dimensions must be between [1, 9]."); + AddOutput("Out", + "(Tensor) The output of clip_by_norm op with shape as input(X)"); + AddAttr("max_norm", "(float) The maximum norm value."); + AddComment(R"DOC( +ClipByNorm operator limits the L2 norm of the input 'X' within 'max_norm'. +If the L2 norm of 'X' is less than or equal to 'max_norm', 'Out' will be +the same as 'X'. If the L2 norm of 'X' is greater than 'max_norm', 'X' will +be linearly scaled to make the L2 norm of 'Out' equal to 'max_norm', as +shown in the following formula： + +'Out' = 'max_norm' * 'X' / norm('X'), + +where norm('X') represents the L2 norm of 'X'. +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(clip_by_norm, ops::ClipByNormOp, + ops::ClipByNormOpMaker); +REGISTER_OP_CPU_KERNEL( + clip_by_norm, ops::ClipByNormKernel); diff --git a/paddle/operators/increment_op.cu b/paddle/operators/clip_by_norm_op.cu similarity index 64% rename from paddle/operators/increment_op.cu rename to paddle/operators/clip_by_norm_op.cu index f97a6c468522f..2593a24ebbf56 100644 --- a/paddle/operators/increment_op.cu +++ b/paddle/operators/clip_by_norm_op.cu @@ -12,11 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/increment_op.h" +#include "paddle/operators/clip_by_norm_op.h" +namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL( - increment, - paddle::operators::IncrementKernel, - paddle::operators::IncrementKernel, - paddle::operators::IncrementKernel, - paddle::operators::IncrementKernel); + clip_by_norm, ops::ClipByNormKernel); diff --git a/paddle/operators/clip_by_norm_op.h b/paddle/operators/clip_by_norm_op.h new file mode 100644 index 0000000000000..b26476cae9b5b --- /dev/null +++ b/paddle/operators/clip_by_norm_op.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/transform.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; + +template +class ClipByNormKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto max_norm = context.Attr("max_norm"); + auto* input = context.Input("X"); + auto* output = context.Output("Out"); + output->mutable_data(context.GetPlace()); + + auto x = EigenVector::Flatten(*input); + auto out = EigenVector::Flatten(*output); + auto x_norm = x.square().sum().sqrt(); + auto place = context.GetEigenDevice(); + + auto temp = (x_norm <= max_norm).template cast().eval(); + auto scaling = temp + (static_cast(1) - temp) * max_norm / x_norm; + Eigen::array one_dim{{1}}; + Eigen::DSizes m_dsize(input->numel()); + out.device(place) = x * scaling.reshape(one_dim).broadcast(m_dsize); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/compare_op.cc b/paddle/operators/compare_op.cc index 8b425d14df3bc..716b5ee92d0d8 100644 --- a/paddle/operators/compare_op.cc +++ b/paddle/operators/compare_op.cc @@ -14,6 +14,7 @@ #include "paddle/operators/compare_op.h" #include "paddle/framework/op_registry.h" + namespace paddle { namespace operators { template @@ -61,19 +62,34 @@ class CompareOpInferShape : public framework::InferShapeBase { } }; +class CompareOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + framework::OpKernelType kt = OperatorWithKernel::GetKernelType(ctx); + // CompareOp kernel's device type is decided by input tensor place + kt.place_ = ctx.Input("X")->place(); + return kt; + } +}; + } // namespace operators } // namespace paddle -#define REGISTER_LOGICAL_OP(op_type, _equation) \ - struct _##op_type##Comment { \ - static char type[]; \ - static char equation[]; \ - }; \ - char _##op_type##Comment::type[]{#op_type}; \ - char _##op_type##Comment::equation[]{_equation}; \ - REGISTER_OP_WITH_KERNEL( \ - op_type, ::paddle::operators::CompareOpProtoMaker<_##op_type##Comment>, \ - ::paddle::operators::CompareOpInferShape<_##op_type##Comment>, \ +#define REGISTER_LOGICAL_OP(op_type, _equation) \ + struct _##op_type##Comment { \ + static char type[]; \ + static char equation[]; \ + }; \ + char _##op_type##Comment::type[]{#op_type}; \ + char _##op_type##Comment::equation[]{_equation}; \ + REGISTER_OPERATOR( \ + op_type, ::paddle::operators::CompareOp, \ + ::paddle::operators::CompareOpProtoMaker<_##op_type##Comment>, \ + ::paddle::operators::CompareOpInferShape<_##op_type##Comment>, \ ::paddle::framework::EmptyGradOpMaker); REGISTER_LOGICAL_OP(less_than, "Out = X < Y"); diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc new file mode 100644 index 0000000000000..d5b124682d755 --- /dev/null +++ b/paddle/operators/conditional_block_op.cc @@ -0,0 +1,197 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include +#include "paddle/framework/executor.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class ConditionalOp : public framework::OperatorBase { + public: + ConditionalOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + protected: + std::vector InputTensors( + const framework::Scope &scope) const { + std::vector retv; + auto xs = Inputs("X"); + retv.resize(xs.size(), nullptr); + std::transform( + xs.begin(), xs.end(), retv.begin(), + [&scope](const std::string &var_name) -> const framework::LoDTensor * { + auto *var = scope.FindVar(var_name); + PADDLE_ENFORCE(var != nullptr, "Cannot find variable %s", var_name); + return &var->Get(); + }); + return retv; + } +}; + +class ConditionalBlockOp : public ConditionalOp { + public: + ConditionalBlockOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ConditionalOp(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto xs = InputTensors(scope); + bool need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); + + if (need_run) { + auto *scope_var = scope.FindVar(Output("Scope")); + PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); + auto *scopes = scope_var->GetMutable>(); + scopes->resize(1); + scopes->front() = &scope.NewScope(); + auto &cur_scope = *scopes->front(); + + auto *block = Attr("block"); + framework::Executor exec(dev_ctx); + exec.Run(*block->Program(), &cur_scope, block->ID(), false); + } + } +}; + +class ConditionalBlockOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ConditionalBlockOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The conditional variable of this operator. If X is empty, the " + "whole sub-block will not be executed.") + .AsDuplicable(); + AddInput("Params", "The input variables of the sub-block.").AsDuplicable(); + AddOutput("Out", "The output variables of the sub-block.").AsDuplicable(); + AddOutput("Scope", + "(std::vector) The step scope of conditional block. To " + "unify the conditional block, rnn and while op, the type of " + "scope is std::vector"); + AddAttr( + "block", "The step block of conditional block operator"); + AddComment(R"DOC(Conditional block operator + +Run the sub-block if X is not empty. Params is the other inputs and Out is the +outputs of the sub-block. +)DOC"); + } +}; + +class ConditionalBlockGradOp : public ConditionalOp { + public: + ConditionalBlockGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ConditionalOp(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto xs = this->InputTensors(scope); + bool need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); + + if (need_run) { + auto *scope_var = scope.FindVar(Input("Scope")); + PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); + auto &scopes = scope_var->Get>(); + framework::Scope &cur_scope = *scopes[0]; + + auto *block = Attr("block"); + framework::Executor exec(dev_ctx); + exec.Run(*block->Program(), &cur_scope, block->ID(), false); + + AssignLocalGradientToGlobal(dev_ctx, cur_scope, Inputs("Params"), + Outputs(framework::GradVarName("Params"))); + + AssignLocalGradientToGlobal(dev_ctx, cur_scope, Inputs("X"), + Outputs(framework::GradVarName("X"))); + } + } + + private: + void AssignLocalGradientToGlobal( + const platform::DeviceContext &dev_ctx, const framework::Scope &cur_scope, + const std::vector &p_names, + const std::vector &pg_names) const { + for (size_t i = 0; i < p_names.size(); ++i) { + auto out_grad_name = pg_names[i]; + auto in_grad_name = framework::GradVarName(p_names[i]); + auto *in_var = cur_scope.FindVar(in_grad_name); + if (in_var == nullptr) { + continue; + } + auto new_in_grad_name = cur_scope.Rename(in_grad_name); + auto assign = + framework::OpRegistry::CreateOp("assign", {{"X", {new_in_grad_name}}}, + {{"Out", {out_grad_name}}}, {}); + assign->Run(cur_scope, dev_ctx); + cur_scope.Rename(new_in_grad_name, in_grad_name); + } + } +}; + +class ConditionalBlockGradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInputs("X")); + if (context->HasInputs("Params")) { + PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("Params"))); + context->SetOutputsDim(framework::GradVarName("Params"), + context->GetInputsDim("Params")); + } + PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("X"))); + context->SetOutputsDim(framework::GradVarName("X"), + context->GetInputsDim("X")); + } +}; + +class ConditionalBlockGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto grad_op = new framework::OpDescBind(); + grad_op->SetType("conditional_block_grad"); + grad_op->SetInput("X", Input("X")); + grad_op->SetInput("Params", Input("Params")); + grad_op->SetInput("Out", Output("Out")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetInput("Scope", Output("Scope")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + grad_op->SetOutput(framework::GradVarName("Params"), InputGrad("Params")); + grad_op->SetBlockAttr("block", *this->grad_block_[0]); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(conditional_block, ops::ConditionalBlockOp, + ops::ConditionalBlockOpProtoMaker, + ops::ConditionalBlockGradMaker); +REGISTER_OPERATOR(conditional_block_grad, ops::ConditionalBlockGradOp, + ops::ConditionalBlockGradInferShape); diff --git a/paddle/operators/expand_op.cc b/paddle/operators/expand_op.cc new file mode 100644 index 0000000000000..282775fcda45f --- /dev/null +++ b/paddle/operators/expand_op.cc @@ -0,0 +1,136 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/expand_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class ExpandOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null."); + + std::vector expand_times = + ctx->Attrs().Get>("expand_times"); + auto x_dims = ctx->GetInputDim("X"); + + PADDLE_ENFORCE_EQ(static_cast(x_dims.size()), expand_times.size(), + "The number of Attr(expand_times)'s value must be equal " + "to the rank of Input(X)."); + PADDLE_ENFORCE_LE(x_dims.size(), 6, + "The rank of Input(X) must not be greater than 6."); + + std::vector out_shape(x_dims.size()); + for (size_t i = 0; i < expand_times.size(); ++i) { + PADDLE_ENFORCE_GE(expand_times[i], 1, + "Each value of Attr(expand_times) should not be " + "less than 1."); + out_shape[i] = x_dims[i] * expand_times[i]; + } + + ctx->SetOutputDim("Out", framework::make_ddim(out_shape)); + if (out_shape[0] == x_dims[0]) { + ctx->ShareLoD("X", "Out"); + } + } +}; + +class ExpandOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ExpandOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(Tensor, default Tensor) A tensor with rank in [1, 6]." + "X is the input tensor to be expanded."); + AddOutput("Out", + "(Tensor, default Tensor) A tensor with rank in [1, 6]." + "The rank of Output(Out) is same as Input(X) except that each " + "dimension size of Output(Out) is equal to corresponding " + "dimension size of Input(X) multiplying corresponding value of " + "Attr(expand_times)."); + AddAttr>("expand_times", + "Expand times number for each dimension."); + AddComment(R"DOC( +Expand operator tiles the input by given times number. You should set times +number for each dimension by providing attribute 'expand_times'. The rank of X +should be in [1, 6]. Please notice that size of 'expand_times' must be same with +X's rank. Following is a using case: + +Input(X) is a 3-D tensor with shape [2, 3, 1]: + + [ + [[1], [2], [3]], + [[4], [5], [6]] + ] + +Attr(expand_times): [1, 2, 2] + +Output(Out) is a 3-D tensor with shape [2, 6, 2]: + + [ + [[1, 1], [2, 2], [3, 3], [1, 1], [2, 2], [3, 3]], + [[4, 4], [5, 5], [6, 6], [4, 4], [5, 5], [6, 6]] + ] + +)DOC"); + } +}; + +class ExpandGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null."); + + auto x_dims = ctx->GetInputDim("X"); + std::vector expand_times = + ctx->Attrs().Get>("expand_times"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); + + for (size_t i = 0; i < expand_times.size(); ++i) { + PADDLE_ENFORCE_EQ(x_dims[i] * expand_times[i], out_dims[i], + "Each dimension size of Input(Out@GRAD) should be " + "equal to multiplication of crroresponding dimension " + "size of Input(X) and Attr(expand_times) value."); + } + + auto x_grad_name = framework::GradVarName("X"); + + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(expand, ops::ExpandOp, ops::ExpandOpMaker, expand_grad, + ops::ExpandGradOp); +REGISTER_OP_CPU_KERNEL(expand, + ops::ExpandKernel); +REGISTER_OP_CPU_KERNEL( + expand_grad, ops::ExpandGradKernel); diff --git a/paddle/operators/fill_constant_op.cu b/paddle/operators/expand_op.cu similarity index 65% rename from paddle/operators/fill_constant_op.cu rename to paddle/operators/expand_op.cu index bca402a8b988b..6744562b6c21d 100644 --- a/paddle/operators/fill_constant_op.cu +++ b/paddle/operators/expand_op.cu @@ -13,12 +13,11 @@ limitations under the License. */ #define EIGEN_USE_GPU -#include "paddle/framework/op_registry.h" -#include "paddle/operators/fill_constant_op.h" + +#include "paddle/operators/expand_op.h" namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(expand, + ops::ExpandKernel); REGISTER_OP_GPU_KERNEL( - fill_constant, ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel); + expand_grad, ops::ExpandGradKernel); diff --git a/paddle/operators/expand_op.h b/paddle/operators/expand_op.h new file mode 100644 index 0000000000000..8ae2c11a5d31d --- /dev/null +++ b/paddle/operators/expand_op.h @@ -0,0 +1,172 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/framework/operator.h" + +#define MAX_RANK_SUPPORTED 6 + +#define EXPAND_TEMPLATE(z, n, data) \ + case n + 1: { \ + Expand(context); \ + break; \ + } +#define REP_EXPAND_TEMPLATE(n) BOOST_PP_REPEAT(n, EXPAND_TEMPLATE, ~) +#define COND(n) \ + BOOST_PP_GREATER_EQUAL(BOOST_PP_DIV(n, MAX_RANK_SUPPORTED), \ + BOOST_PP_MOD(n, MAX_RANK_SUPPORTED)) +#define EXPAND_GRAD_CASE(n) \ + case n: { \ + ExpandBackward(context, reshape_dims_vec, reduce_dims_vec); \ + break; \ + } +#define EXPAND_GRAD_TEMPLATE(z, n, data) \ + BOOST_PP_IF(COND(n), EXPAND_GRAD_CASE(n), ) +#define REP_EXPAND_GRAD_TEMPLATE(n) BOOST_PP_REPEAT(n, EXPAND_GRAD_TEMPLATE, ~) + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; +template +using EigenTensor = framework::EigenTensor; + +template +class ExpandKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto rank = context.Input("X")->dims().size(); + switch (rank) { + REP_EXPAND_TEMPLATE(MAX_RANK_SUPPORTED) + default: + PADDLE_ENFORCE(false, + "Only support tensor with rank being between 1 and 6."); + } + } + + protected: + template + void Expand(const framework::ExecutionContext& context) const { + auto* in0 = context.Input("X"); + auto& expand_times = context.Attr>("expand_times"); + auto* out0 = context.Output("Out"); + Eigen::DSizes bcast_dims; + auto x_dims = in0->dims(); + for (size_t i = 0; i < expand_times.size(); ++i) { + bcast_dims[i] = expand_times[i]; + } + auto x = EigenTensor::From(*in0); + out0->mutable_data(context.GetPlace()); + auto y = EigenTensor::From(*out0); + auto place = context.GetEigenDevice(); + y.device(place) = x.broadcast(bcast_dims); + } +}; + +template +class ExpandGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto& expand_times = context.Attr>("expand_times"); + auto x_dims = in0->dims(); + // 1. reshape_dims_vec is the broadcast parameter. For each dimension i, + // if expand_times[i] > 1 and x_dims[i] > 1, i will be splitted to two + // dimensions [expand_times[i], x_dims[i]]. + // 2. reduce_dims_vec is the dimension parameter to compute gradients. For + // each dimension expanded, the gradients should be summed to original + // size. + std::vector reshape_dims_vec; + std::vector reduce_dims_vec; + for (size_t i = 0; i < expand_times.size(); ++i) { + if (expand_times[i] == 1) { + reshape_dims_vec.push_back(x_dims[i]); + } else { + if (x_dims[i] == 1) { + reduce_dims_vec.push_back(reshape_dims_vec.size()); + reshape_dims_vec.push_back(expand_times[i]); + } else { + reduce_dims_vec.push_back(reshape_dims_vec.size()); + reshape_dims_vec.push_back(expand_times[i]); + reshape_dims_vec.push_back(x_dims[i]); + } + } + } + + int dims = reshape_dims_vec.size() * MAX_RANK_SUPPORTED + + reduce_dims_vec.size() - MAX_RANK_SUPPORTED - 1; + // no need reduce, just copy + if (reduce_dims_vec.size() == 0) { + auto* in0 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + out0->mutable_data(context.GetPlace()); + out0->CopyFrom(*in0, context.GetPlace(), context.device_context()); + } else { + switch (dims) { + REP_EXPAND_GRAD_TEMPLATE(72) + default: + PADDLE_ENFORCE( + false, "Only support tensor with rank being between 1 and 6."); + } + } + } + + protected: + template + void ExpandBackward(const framework::ExecutionContext& context, + const std::vector& reshape_dims_vec, + const std::vector& reduce_dims_vec) const { + size_t reshape_size = Dims / MAX_RANK_SUPPORTED + 1; + size_t reduce_size = Dims % MAX_RANK_SUPPORTED + 1; + PADDLE_ENFORCE_EQ(reshape_size, reshape_dims_vec.size(), + "Inconsistent size between template Dims and " + "reshape dimensions."); + PADDLE_ENFORCE_EQ(reduce_size, reduce_dims_vec.size(), + "Inconsistent size between template Dims and " + "reduce dimensions."); + auto* in0 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + auto x = EigenVector::Flatten(*(context.Input("X"))); + out0->mutable_data(context.GetPlace()); + auto x_grad = EigenVector::Flatten(*out0); + Eigen::DSizes reshape_dims; + for (size_t i = 0; i < reshape_size; ++i) { + reshape_dims[i] = reshape_dims_vec[i]; + } + Eigen::DSizes reduce_dims; + for (size_t i = 0; i < reduce_size; ++i) { + reduce_dims[i] = reduce_dims_vec[i]; + } + auto out_grad = EigenVector::Flatten(*in0); + x_grad.device(context.GetEigenDevice()) = + out_grad.reshape(reshape_dims).sum(reduce_dims).reshape(x.dimensions()); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/fill_constant_batch_size_like_op.cc b/paddle/operators/fill_constant_batch_size_like_op.cc index f86ee3c3d8867..85871ebbfcd8e 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.cc +++ b/paddle/operators/fill_constant_batch_size_like_op.cc @@ -75,10 +75,10 @@ class FillConstantBatchSizeLikeOpMaker "with the specified value"); AddAttr>("shape", "(vector) The shape of the output"); AddAttr("input_dim_idx", - "(int, default 0) the index of input's batch size dimension") + "(int, default 0) The index of input's batch size dimension") .SetDefault(0); AddAttr("output_dim_idx", - "(int, default 0) the index of output's batch size dimension") + "(int, default 0) The index of output's batch size dimension") .SetDefault(0); AddAttr("value", "(float, default 0) The value to be filled") .SetDefault(0.0f); diff --git a/paddle/operators/fill_constant_batch_size_like_op.cu b/paddle/operators/fill_constant_batch_size_like_op.cu index cfa5df001e9d6..298c196f1dfef 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.cu +++ b/paddle/operators/fill_constant_batch_size_like_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/framework/op_registry.h" #include "paddle/operators/fill_constant_batch_size_like_op.h" diff --git a/paddle/operators/fill_constant_batch_size_like_op.h b/paddle/operators/fill_constant_batch_size_like_op.h index a360e6683ec72..339d97a30a581 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.h +++ b/paddle/operators/fill_constant_batch_size_like_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { @@ -27,9 +27,8 @@ class FillConstantBatchSizeLikeOpKernel : public framework::OpKernel { out->mutable_data(ctx.GetPlace()); auto value = ctx.Attr("value"); - auto out_eigen = framework::EigenVector::Flatten(*out); - auto place = ctx.GetEigenDevice(); - out_eigen.device(place) = out_eigen.constant(static_cast(value)); + math::SetConstant setter; + setter(ctx.device_context(), out, static_cast(value)); } }; diff --git a/paddle/operators/fill_constant_op.cc b/paddle/operators/fill_constant_op.cc index 5a1cba51f83bb..818f113b90a4c 100644 --- a/paddle/operators/fill_constant_op.cc +++ b/paddle/operators/fill_constant_op.cc @@ -12,33 +12,41 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/fill_constant_op.h" +#include "paddle/framework/data_type.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { -class FillConstantOp : public framework::OperatorWithKernel { +class FillConstantInferShape : public framework::InferShapeBase { public: - using framework::OperatorWithKernel::OperatorWithKernel; - - void InferShape(framework::InferShapeContext *ctx) const override { + void operator()(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of FillConstantOp should not be null."); auto &shape = ctx->Attrs().Get>("shape"); - std::vector shape_int64(shape.size(), 0); - std::transform(shape.begin(), shape.end(), shape_int64.begin(), - [](int a) { return static_cast(a); }); - auto dims = framework::make_ddim(shape_int64); - ctx->SetOutputDim("Out", dims); + ctx->SetOutputDim("Out", framework::make_ddim(shape)); } +}; - protected: - framework::OpKernelType GetKernelType( - const framework::ExecutionContext &ctx) const override { - int data_type = ctx.Attr("data_type"); - VLOG(10) << " FillConstant data_type = " << data_type; - return framework::OpKernelType(static_cast(data_type), - ctx.device_context()); +class FillConstantOp : public framework::OperatorBase { + public: + using framework::OperatorBase::OperatorBase; + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto data_type = static_cast(Attr("data_type")); + auto value = Attr("value"); + auto force_cpu = Attr("force_cpu"); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + out.Resize(framework::make_ddim(Attr>("shape"))); + if (force_cpu) { + auto cpu = platform::CPUPlace(); + out.mutable_data(cpu, framework::ToTypeIndex(data_type)); + } else { + out.mutable_data(dev_ctx.GetPlace(), framework::ToTypeIndex(data_type)); + } + math::set_constant(dev_ctx, &out, value); } }; @@ -54,6 +62,11 @@ class FillConstantOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr>("shape", "(vector) The shape of the output"); AddAttr("value", "(float, default 0) The value to be filled") .SetDefault(0.0f); + AddAttr("force_cpu", + "(bool, default false) Force fill output variable to cpu " + "memory. Otherwise, fill output variable to the running " + "device") + .SetDefault(false); AddOutput("Out", "(Tensor) Tensor of specified shape will be filled " "with the specified value"); @@ -69,10 +82,6 @@ Fill up a variable with specified constant value. } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_WITHOUT_GRADIENT(fill_constant, ops::FillConstantOp, - ops::FillConstantOpMaker); -REGISTER_OP_CPU_KERNEL( - fill_constant, ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel); +REGISTER_OPERATOR(fill_constant, ops::FillConstantOp, + ops::FillConstantInferShape, ops::FillConstantOpMaker, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/fill_constant_op.h b/paddle/operators/fill_constant_op.h deleted file mode 100644 index 3668f42f1c295..0000000000000 --- a/paddle/operators/fill_constant_op.h +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -template -class FillConstantOpKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - auto* out = ctx.Output("Out"); - out->mutable_data(ctx.GetPlace()); - auto value = ctx.Attr("value"); - - auto out_eigen = framework::EigenVector::Flatten(*out); - auto place = ctx.GetEigenDevice(); - out_eigen.device(place) = out_eigen.constant(static_cast(value)); - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/fill_zeros_like_op.cu b/paddle/operators/fill_zeros_like_op.cu index fdbcf520a0d7b..a6d4ba64bde53 100644 --- a/paddle/operators/fill_zeros_like_op.cu +++ b/paddle/operators/fill_zeros_like_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/framework/op_registry.h" #include "paddle/operators/fill_zeros_like_op.h" diff --git a/paddle/operators/fill_zeros_like_op.h b/paddle/operators/fill_zeros_like_op.h index cdf56a723b117..7e7d78eea2bce 100644 --- a/paddle/operators/fill_zeros_like_op.h +++ b/paddle/operators/fill_zeros_like_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { @@ -23,10 +23,11 @@ template class FillZerosLikeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* output = context.Output("Y"); - output->mutable_data(context.GetPlace()); - auto t = framework::EigenVector::Flatten(*output); - t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + auto* out = context.Output("Y"); + out->mutable_data(context.GetPlace()); + + math::SetConstant setter; + setter(context.device_context(), out, static_cast(0)); } }; diff --git a/paddle/operators/increment_op.cc b/paddle/operators/increment_op.cc index deb02bf2bf82a..35efb12932f1d 100644 --- a/paddle/operators/increment_op.cc +++ b/paddle/operators/increment_op.cc @@ -12,22 +12,57 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/increment_op.h" +#include "paddle/framework/op_registry.h" namespace paddle { namespace operators { -class IncrementOp : public framework::OperatorWithKernel { +class IncrementInferShape : public framework::InferShapeBase { public: - using framework::OperatorWithKernel::OperatorWithKernel; - - void InferShape(framework::InferShapeContext *ctx) const override { + void operator()(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of IncrementOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of IncrementOp should not be null."); + PADDLE_ENFORCE_EQ(1, framework::product(ctx->GetInputDim("X"))); ctx->SetOutputDim("Out", ctx->GetInputDim("X")); - ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +struct IncrementFunctor { + IncrementFunctor(const framework::LoDTensor &x, framework::LoDTensor *out, + float value) + : x_(x), out_(out), value_(value) {} + + template + void operator()() const { + *out_->data() = *x_.data() + static_cast(value_); + } + + const framework::LoDTensor &x_; + framework::LoDTensor *out_; + float value_; +}; + +class IncrementOp : public framework::OperatorBase { + public: + IncrementOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + + PADDLE_ENFORCE(platform::is_cpu_place(x.place())); + out.Resize(x.dims()); + out.mutable_data(x.place(), x.type()); + float value = Attr("step"); + framework::VisitDataType(framework::ToDataType(out.type()), + IncrementFunctor(x, &out, value)); } }; @@ -59,10 +94,10 @@ class IncrementGradOpMaker : public framework::SingleGradOpDescMaker { std::unique_ptr Apply() const override { auto *grad_op = new framework::OpDescBind(); - grad_op->SetType("scale"); - grad_op->SetInput("X", OutputGrad("Out")); - grad_op->SetOutput("Out", InputGrad("X")); - grad_op->SetAttr("scale", 1.0f); + grad_op->SetType("increment"); + grad_op->SetInput("X", Output("Out")); + grad_op->SetOutput("Out", Input("X")); + grad_op->SetAttr("step", -boost::get(GetAttr("step"))); return std::unique_ptr(grad_op); } }; @@ -71,11 +106,5 @@ class IncrementGradOpMaker : public framework::SingleGradOpDescMaker { } // namespace paddle namespace ops = paddle::operators; - -REGISTER_OPERATOR(increment, ops::IncrementOp, ops::IncrementOpMaker, - ops::IncrementGradOpMaker); -REGISTER_OP_CPU_KERNEL( - increment, ops::IncrementKernel, - ops::IncrementKernel, - ops::IncrementKernel, - ops::IncrementKernel); +REGISTER_OPERATOR(increment, ops::IncrementOp, ops::IncrementInferShape, + ops::IncrementOpMaker, ops::IncrementGradOpMaker); diff --git a/paddle/operators/increment_op.h b/paddle/operators/increment_op.h deleted file mode 100644 index 3d53256dd1a27..0000000000000 --- a/paddle/operators/increment_op.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#pragma once - -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { -template -class IncrementKernel : public framework::OpKernel { - public: - virtual void Compute(const framework::ExecutionContext& context) const { - auto* tensor = context.Output("Out"); - auto* in = context.Input("X"); - tensor->mutable_data(in->place()); - - auto step = static_cast(context.Attr("step")); - - auto eigen_out = framework::EigenVector::Flatten(*tensor); - auto eigen_in = framework::EigenVector::Flatten(*in); - auto& place = context.GetEigenDevice(); - eigen_out.device(place) = eigen_in + step; - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/l1_norm_op.h b/paddle/operators/l1_norm_op.h index de459818ad83d..3c60dc3dc7415 100644 --- a/paddle/operators/l1_norm_op.h +++ b/paddle/operators/l1_norm_op.h @@ -29,7 +29,7 @@ class L1NormKernel : public framework::OpKernel { Out->mutable_data(context.GetPlace()); auto x = framework::EigenVector::Flatten(*X); - auto out = framework::EigenVector::Flatten(*Out); + auto out = framework::EigenScalar::From(*Out); auto place = context.GetEigenDevice(); out.device(place) = x.abs().sum(); diff --git a/paddle/operators/lod_array_length_op.cc b/paddle/operators/lod_array_length_op.cc new file mode 100644 index 0000000000000..80445eb575703 --- /dev/null +++ b/paddle/operators/lod_array_length_op.cc @@ -0,0 +1,71 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class LoDArrayLengthOp : public framework::OperatorBase { + public: + LoDArrayLengthOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + out.Resize({1}); + auto cpu = platform::CPUPlace(); + *out.mutable_data(cpu) = static_cast(x.size()); + } +}; + +class LoDArrayLengthProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDArrayLengthProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(LoDTensorArray) The input tensor array."); + AddOutput("Out", "(Tensor) 1x1 CPU Tensor of length, int64_t"); + AddComment(R"DOC(Get the length of lod tensor array + +Out = len(X) + +NOTE: The output is a CPU Tensor since the control variable should be only in +CPU and the length of LoDTensorArray should be used as control variables. +)DOC"); + } +}; + +class LoDArrayLengthInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X")); + PADDLE_ENFORCE(context->HasOutput("Out")); + context->SetOutputDim("Out", {1}); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(lod_array_length, ops::LoDArrayLengthOp, + ops::LoDArrayLengthInferShape, ops::LoDArrayLengthProtoMaker, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/lod_rank_table_op.cc b/paddle/operators/lod_rank_table_op.cc index ce010fcb91873..f7d4db1947b83 100644 --- a/paddle/operators/lod_rank_table_op.cc +++ b/paddle/operators/lod_rank_table_op.cc @@ -66,7 +66,8 @@ class LoDRankTableInferVarType : public framework::VarTypeInference { void operator()(const framework::OpDescBind &op_desc, framework::BlockDescBind *block) const override { for (auto &o : op_desc.Output("Out")) { - block->Var(o)->SetType(framework::VarDesc::LOD_RANK_TABLE); + block->FindRecursiveOrCreateVar(o)->SetType( + framework::VarDesc::LOD_RANK_TABLE); } } }; diff --git a/paddle/operators/lod_reset_op.cc b/paddle/operators/lod_reset_op.cc new file mode 100644 index 0000000000000..32831cb1e2cf1 --- /dev/null +++ b/paddle/operators/lod_reset_op.cc @@ -0,0 +1,120 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/lod_reset_op.h" + +namespace paddle { +namespace operators { + +class LoDResetOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + // input check + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of LoDResetOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of LoDResetOp should not be null."); + // If target LoD is not set form Input(), then it must be set from Attr(). + if (!ctx->HasInput("TargetLoD")) { + auto level0 = ctx->Attrs().Get>("target_lod"); + PADDLE_ENFORCE(level0.size() > 1, + "Target LoD is not found, should be set to be a valid one " + "through Input() or Attr()."); + } + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); + } +}; + +class LoDResetOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDResetOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(LoDTensor) The input tensor of lod_reset operator."); + AddInput("TargetLoD", + "(Tensor, optional) The target level 0 LoD from Input().") + .AsDispensable(); + AddOutput("Out", "(LoDTensor) The output tensor of lod_reset operator."); + AddAttr>("target_lod", + "The target level 0 LoD from Attr().") + .SetDefault(std::vector{}); + AddComment(R"DOC(LoDReset operator + +Reset LoD of Input(X) into a new one specified by Input(TargetLoD) or +Attr(target_lod), or set LoD for Input(X) if it doesn't have one. +Currently the lod_reset operator only supports the reset of level 0 LoD. +At least one of Input(TargetLoD) and Attr(target_lod) must be set, +and if both of them are set, Input(TargetLoD) will be chosen as the +target LoD. + +An example: +Given a float LoDTensor X with shape (6, 1), its transpose form represents + + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + +with LoD = [[0, 2, 5, 6]] and the three (transposed) sequences look like + + [1.0, 2.0], [3.0, 4.0, 5.0], [6.0]. + +If target LoD = [0, 4, 6], the lod_reset operator will reset the LoD and +the sequences that the LoDTensor Output(Out) contains becomes: + + [1.0, 2.0, 3.0, 4.0], [5.0, 6.0]. + +)DOC"); + } +}; + +class LoDResetGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(lod_reset, ops::LoDResetOp, ops::LoDResetOpMaker, lod_reset_grad, + ops::LoDResetGradOp); +REGISTER_OP_CPU_KERNEL(lod_reset, + ops::LoDResetKernel, + ops::LoDResetKernel); +REGISTER_OP_CPU_KERNEL( + lod_reset_grad, ops::LoDResetGradKernel, + ops::LoDResetGradKernel); diff --git a/paddle/operators/lod_reset_op.cu b/paddle/operators/lod_reset_op.cu new file mode 100644 index 0000000000000..5244a17c3aad0 --- /dev/null +++ b/paddle/operators/lod_reset_op.cu @@ -0,0 +1,24 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/lod_reset_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(lod_reset, + ops::LoDResetKernel, + ops::LoDResetKernel); +REGISTER_OP_GPU_KERNEL( + lod_reset_grad, ops::LoDResetGradKernel, + ops::LoDResetGradKernel); diff --git a/paddle/operators/lod_reset_op.h b/paddle/operators/lod_reset_op.h new file mode 100644 index 0000000000000..2bb916ccee80c --- /dev/null +++ b/paddle/operators/lod_reset_op.h @@ -0,0 +1,78 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class LoDResetKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* out = ctx.Output("Out"); + auto* in = ctx.Input("X"); + auto* lod_t = ctx.Input("TargetLoD"); + + std::vector level0; + if (lod_t) { + auto* lod = lod_t->data(); + if (platform::is_gpu_place(ctx.GetPlace())) { + framework::Tensor lod_cpu; + lod_cpu.CopyFrom(*lod_t, platform::CPUPlace(), ctx.device_context()); + lod = lod_cpu.data(); + } + level0 = std::vector(lod, lod + lod_t->numel()); + } else { + level0 = ctx.Attr>("target_lod"); + } + + PADDLE_ENFORCE(level0.size() > 1UL, + "The size of target LoD should be greater than 1."); + PADDLE_ENFORCE(level0[0] == 0, + "Target LoD should be a vector starting from 0."); + PADDLE_ENFORCE(level0.back() == in->dims()[0], + "Target LoD should be a vector end with the " + "first dimension of Input(X)."); + for (size_t i = 0; i < level0.size() - 1; ++i) { + PADDLE_ENFORCE(level0[i + 1] > level0[i], + "Target LoD should be an ascending vector."); + } + + out->ShareDataWith(*in); + // cast level0 to size_t + std::vector ulevel0(level0.size(), 0); + std::transform(level0.begin(), level0.end(), ulevel0.begin(), + [](int a) { return static_cast(a); }); + framework::LoD target_lod; + target_lod.push_back(ulevel0); + out->set_lod(target_lod); + } +}; + +template +class LoDResetGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* d_out = ctx.Input(framework::GradVarName("Out")); + auto* d_x = ctx.Output(framework::GradVarName("X")); + + d_x->ShareDataWith(*d_out); + } +}; +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/lod_tensor_to_array_op.cc b/paddle/operators/lod_tensor_to_array_op.cc index 5f02f5e8a1283..58af35564d83b 100644 --- a/paddle/operators/lod_tensor_to_array_op.cc +++ b/paddle/operators/lod_tensor_to_array_op.cc @@ -133,6 +133,22 @@ class LoDTensorToArrayInferVarType : public framework::VarTypeInference { } }; +class LoDTensorToArrayGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("array_to_lod_tensor"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetInput("RankTable", Input("RankTable")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + } // namespace operators } // namespace paddle @@ -140,4 +156,5 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(lod_tensor_to_array, ops::LoDTensorToArrayOp, ops::LoDTensorToArrayOpProtoMaker, ops::LoDTensorToArrayInferShape, - ops::LoDTensorToArrayInferVarType); + ops::LoDTensorToArrayInferVarType, + ops::LoDTensorToArrayGradMaker); diff --git a/paddle/operators/lstm_op.cc b/paddle/operators/lstm_op.cc index 6b859dbbe7f76..4cbb60f3fdab9 100644 --- a/paddle/operators/lstm_op.cc +++ b/paddle/operators/lstm_op.cc @@ -24,6 +24,11 @@ class LSTMOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Input"), "Input(Input) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Bias"), + "Input(Bias) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Hidden"), "Output(Hidden) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Cell"), @@ -59,11 +64,13 @@ class LSTMOp : public framework::OperatorWithKernel { "The second dimension of Input(Weight) " "should be 4 * %d.", frame_size); + auto b_dims = ctx->GetInputDim("Bias"); PADDLE_ENFORCE_EQ(b_dims.size(), 2, "The rank of Input(Bias) should be 2."); PADDLE_ENFORCE_EQ(b_dims[0], 1, "The first dimension of Input(Bias) should be 1."); - if (ctx->Attrs().Get("usePeepholes")) { + + if (ctx->Attrs().Get("use_peepholes")) { PADDLE_ENFORCE_EQ(b_dims[1], 7 * frame_size, "The second dimension of Input(Bias) should be " "7 * %d if enable peepholes connection", @@ -74,6 +81,7 @@ class LSTMOp : public framework::OperatorWithKernel { "4 * %d if disable peepholes connection", frame_size); } + framework::DDim out_dims({in_dims[0], frame_size}); ctx->SetOutputDim("Hidden", out_dims); ctx->SetOutputDim("Cell", out_dims); @@ -118,14 +126,13 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Bias", "(Tensor) the learnable weights, which contains two parts: " "input-hidden bias weight and peephole connections weight if " - "setting `usePeepholes` True. " - "1. `usePeepholes = False` " + "setting `use_peepholes` True. " + "1. `use_peepholes = False` " " - The shape is (1 x 4D). " " - Bias = {b_c, b_i, b_f, b_o}." - "2. `usePeepholes = True` " + "2. `use_peepholes = True` " " - The shape is (1 x 7D). " - " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}.") - .AsDispensable(); + " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}."); AddOutput("Hidden", "(LoDTensor) the hidden state of LSTM operator. " "The shape is (T x D), and lod is the same with the `Input`."); @@ -145,29 +152,32 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { "(LoDTensor) This LoDTensor is obtained in the forward and used " "in the backward.") .AsIntermediate(); - AddAttr("usePeepholes", - "(bool, default True) " + AddAttr("use_peepholes", + "(bool, defalut: True) " "whether to enable diagonal/peephole connections.") .SetDefault(true); - AddAttr("isReverse", - "(bool, default False) " + AddAttr("is_reverse", + "(bool, defalut: False) " "whether to compute reversed LSTM.") .SetDefault(false); AddAttr( - "gateActivation", - "(string, default sigmoid)" + "gate_activation", + "(string, default: sigmoid)" "The activation for input gate, forget gate and output " "gate, `sigmoid` by default.") - .SetDefault("sigmoid"); - AddAttr("cellActivation", - "(string, default tanh)" + .SetDefault("sigmoid") + .InEnum({"sigmoid", "tanh", "relu", "identity"}); + AddAttr("cell_activation", + "(string, default: tanh)" "The activation for cell output, `tanh` by defalut.") - .SetDefault("tanh"); - AddAttr("candidateActivation", - "(string, default tanh)" + .SetDefault("tanh") + .InEnum({"sigmoid", "tanh", "relu", "identity"}); + AddAttr("candidate_activation", + "(string, default: tanh)" "The activation for candidate hidden state, " "`tanh` by default.") - .SetDefault("tanh"); + .SetDefault("tanh") + .InEnum({"sigmoid", "tanh", "relu", "identity"}); AddComment(R"DOC( Long-Short Term Memory (LSTM) Operator. @@ -203,7 +213,7 @@ are the cell input and cell output activation functions and `tanh` is usually used for them. \f$\tilde{c_t}\f$ is also called candidate hidden state, which is computed based on the current input and the previous hidden state. -Set usePeepholes False to disable peephole connection +Set `use_peepholes` False to disable peephole connection (http://www.bioinf.jku.at/publications/older/2604.pdf). The formula is omitted here. @@ -226,23 +236,27 @@ class LSTMGradOp : public framework::OperatorWithKernel { "Input(Hidden) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasInput("Cell"), "Input(Cell) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Bias"), + "Input(Bias) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasInput("BatchGate"), "Input(BatchGate) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasInput("BatchCellPreAct"), "Input(BatchGate) of LSTM should not be null."); - auto in_g_name = framework::GradVarName("Input"); - if (ctx->HasOutput(in_g_name)) - ctx->SetOutputDim(in_g_name, ctx->GetInputDim("Input")); - - auto w_g_name = framework::GradVarName("Weight"); - if (ctx->HasOutput(w_g_name)) - ctx->SetOutputDim(w_g_name, ctx->GetInputDim("Weight")); - - auto b_g_name = framework::GradVarName("Bias"); - if (ctx->HasOutput(b_g_name)) - ctx->SetOutputDim(b_g_name, ctx->GetInputDim("Bias")); + auto SetOutGradDim = [&ctx](const std::string& name) { + auto g_name = framework::GradVarName(name); + if (ctx->HasOutput(g_name)) + ctx->SetOutputDim(g_name, ctx->GetInputDim(name)); + }; + + SetOutGradDim("Input"); + SetOutGradDim("Weight"); + SetOutGradDim("Bias"); + SetOutGradDim("H0"); + SetOutGradDim("C0"); } protected: diff --git a/paddle/operators/lstm_op.h b/paddle/operators/lstm_op.h index af088b80b4283..fca84e2d8fa83 100644 --- a/paddle/operators/lstm_op.h +++ b/paddle/operators/lstm_op.h @@ -28,6 +28,15 @@ template using EigenMatrix = framework::EigenMatrix; +template +inline void ReorderInitState(const platform::DeviceContext& ctx, + const framework::Tensor& src, const size_t* index, + framework::Tensor* dst, bool indexed_src) { + math::CopyMatrixRowsFunctor row_shuffle; + dst->mutable_data(src.dims(), ctx.GetPlace()); + row_shuffle(ctx, src, index, *dst, indexed_src); +} + template class LSTMKernel : public framework::OpKernel { public: @@ -36,6 +45,9 @@ class LSTMKernel : public framework::OpKernel { auto* weight = ctx.Input("Weight"); auto* bias = ctx.Input("Bias"); + auto* hidden_t0 = ctx.Input("H0"); + auto* cell_t0 = ctx.Input("C0"); + auto* batch_gate = ctx.Output("BatchGate"); batch_gate->mutable_data(ctx.GetPlace()); auto* hidden_out = ctx.Output("Hidden"); @@ -43,12 +55,7 @@ class LSTMKernel : public framework::OpKernel { auto* cell_out = ctx.Output("Cell"); cell_out->mutable_data(ctx.GetPlace()); - // Now the function ShareLoD in InferShape is not implemented. - // So copy LoD here. - ctx.ShareLoD("Input", "Hidden"); - ctx.ShareLoD("Input", "Cell"); - - bool is_reverse = ctx.Attr("isReverse"); + bool is_reverse = ctx.Attr("is_reverse"); math::LoDTensor2BatchFunctor to_batch; auto& device_ctx = ctx.device_context(); to_batch(device_ctx, *input, *batch_gate, true, is_reverse); @@ -71,7 +78,7 @@ class LSTMKernel : public framework::OpKernel { } math::LstmMetaValue lstm_value; - if (bias) { + if (bias && ctx.Attr("use_peepholes")) { T* bias_data = const_cast(bias->data()); // the code style in LstmMetaValue will be updated later. @@ -84,6 +91,16 @@ class LSTMKernel : public framework::OpKernel { lstm_value.checkOg = nullptr; } lstm_value.prevStateValue = nullptr; + Tensor ordered_c0; + const size_t* order = batch_gate->lod()[2].data(); + if (cell_t0) { + // Since the batch computing for LSTM reorders the input sequence + // according to their length. The initialized cell state also needs + // to reorder. + ReorderInitState(device_ctx, *cell_t0, order, &ordered_c0, + true); + lstm_value.prevStateValue = ordered_c0.data(); + } // Use the local variable as here. LoDTensor batch_hidden, batch_cell; @@ -94,9 +111,9 @@ class LSTMKernel : public framework::OpKernel { auto batch_starts = batch_gate->lod()[0]; size_t num_batch = batch_starts.size() - 1; - auto gate_act = ctx.Attr("gateActivation"); - auto cell_act = ctx.Attr("cellActivation"); - auto cand_act = ctx.Attr("candidateActivation"); + auto gate_act = ctx.Attr("gate_activation"); + auto cell_act = ctx.Attr("cell_activation"); + auto cand_act = ctx.Attr("candidate_activation"); for (size_t n = 0; n < num_batch; n++) { int bstart = static_cast(batch_starts[n]); @@ -109,15 +126,28 @@ class LSTMKernel : public framework::OpKernel { int cur_batch_size = bend - bstart; - if (n != 0) { + if (n > 0) { int pre_h_start = static_cast(batch_starts[n - 1]); int pre_h_end = pre_h_start + cur_batch_size; auto pre_hidden_t = batch_hidden.Slice(pre_h_start, pre_h_end); math::matmul(device_ctx, pre_hidden_t, false, *weight, false, static_cast(1.0), &gate_t, static_cast(1.0)); + } else if (hidden_t0) { + // If n == 0 and there is no initialized hidden state, that is to say + // the H0 is zeros, the calculation W_h * H0 will be skiped. + // If n == 0 and there is initialized hidden state, calculate W_h * H0. + + // Since the batch computing for LSTM reorders the input sequence + // according to their length. The initialized hidden state also needs + // to reorder. + Tensor ordered_h0; + ReorderInitState(device_ctx, *hidden_t0, order, &ordered_h0, + true); + math::matmul(device_ctx, ordered_h0, false, *weight, false, + static_cast(1.0), &gate_t, + static_cast(1.0)); } - // else if : FIXME support the initial hidden and cell lstm_value.gateValue = gate_t.data(); lstm_value.outputValue = out_t.data(); @@ -160,6 +190,12 @@ class LSTMGradKernel : public framework::OpKernel { auto* weight_g = ctx.Output(framework::GradVarName("Weight")); auto* bias_g = ctx.Output(framework::GradVarName("Bias")); + auto* h0 = ctx.Input("H0"); + auto* c0 = ctx.Input("C0"); + + auto* h0_g = ctx.Output(framework::GradVarName("H0")); + auto* c0_g = ctx.Output(framework::GradVarName("C0")); + auto& device_ctx = ctx.device_context(); math::SetConstant zero; if (weight_g) { @@ -167,13 +203,25 @@ class LSTMGradKernel : public framework::OpKernel { zero(device_ctx, weight_g, static_cast(0.0)); } + // ordered_h0/c0 is the reordered hidden/cell initialization. + // ordered_h0_g/c0_g is the reordered gradient of hidden/cell + // initialization. + Tensor ordered_h0, ordered_c0, ordered_h0_g, ordered_c0_g; + const size_t* order = batch_gate->lod()[2].data(); + if (c0) { + ReorderInitState(device_ctx, *c0, order, &ordered_c0, true); + } + if (c0 && c0_g) { + ordered_c0_g.mutable_data(c0_g->dims(), ctx.GetPlace()); + } + auto in_dims = input->dims(); auto out_dims = hidden_g->dims(); int frame_size = static_cast(in_dims[1] / 4); PADDLE_ENFORCE_EQ(frame_size, out_dims[1]); math::LstmMetaValue lstm_value; - if (bias) { + if (bias && ctx.Attr("use_peepholes")) { T* bias_data = const_cast(bias->data()); lstm_value.checkIg = bias_data + 4 * frame_size; lstm_value.checkFg = lstm_value.checkIg + frame_size; @@ -185,9 +233,13 @@ class LSTMGradKernel : public framework::OpKernel { } math::LstmMetaGrad lstm_grad; + if (bias && bias_g) { - T* bias_g_data = const_cast(bias_g->mutable_data(ctx.GetPlace())); + bias_g->mutable_data(ctx.GetPlace()); zero(device_ctx, bias_g, static_cast(0.0)); + } + if (bias && bias_g && ctx.Attr("use_peepholes")) { + T* bias_g_data = bias_g->data(); lstm_grad.checkIgGrad = bias_g_data + 4 * frame_size; lstm_grad.checkFgGrad = lstm_grad.checkIgGrad + frame_size; lstm_grad.checkOgGrad = lstm_grad.checkFgGrad + frame_size; @@ -199,36 +251,30 @@ class LSTMGradKernel : public framework::OpKernel { math::LoDTensor2BatchFunctor to_batch; - // use the local variable as here. - LoDTensor batch_hidden; - batch_hidden.mutable_data(out_dims, ctx.GetPlace()); - batch_hidden.set_lod(batch_gate->lod()); - to_batch(device_ctx, *hidden_out, batch_hidden, false); + auto ToBatch = [&batch_gate, &to_batch]( + const platform::DeviceContext& ctx, const framework::LoDTensor& src, + const framework::DDim& dims, framework::LoDTensor& dst) { + dst.mutable_data(dims, ctx.GetPlace()); + dst.set_lod(batch_gate->lod()); + to_batch(ctx, src, dst, false); + }; - LoDTensor batch_hidden_g; - batch_hidden_g.mutable_data(out_dims, ctx.GetPlace()); - batch_hidden_g.set_lod(batch_gate->lod()); - to_batch(device_ctx, *hidden_g, batch_hidden_g, false); + LoDTensor batch_hidden, batch_hidden_g, batch_cell; + ToBatch(device_ctx, *hidden_out, out_dims, batch_hidden); + ToBatch(device_ctx, *hidden_g, out_dims, batch_hidden_g); + ToBatch(device_ctx, *cell_out, out_dims, batch_cell); - LoDTensor batch_cell; - batch_cell.mutable_data(out_dims, ctx.GetPlace()); - batch_cell.set_lod(batch_gate->lod()); - to_batch(device_ctx, *cell_out, batch_cell, false); - - LoDTensor batch_cell_g; + LoDTensor batch_cell_g, batch_gate_g; batch_cell_g.mutable_data(out_dims, ctx.GetPlace()); - batch_cell_g.set_lod(batch_gate->lod()); // TODO(qingqing) support the case output cell has gradient. // to_batch(device_ctx, *cell_g, batch_cell_g, false); zero(device_ctx, &batch_cell_g, static_cast(0.0)); - - LoDTensor batch_gate_g; batch_gate_g.mutable_data(batch_gate->dims(), ctx.GetPlace()); batch_gate_g.set_lod(batch_gate->lod()); - auto gate_act = ctx.Attr("gateActivation"); - auto cell_act = ctx.Attr("cellActivation"); - auto cand_act = ctx.Attr("candidateActivation"); + auto gate_act = ctx.Attr("gate_activation"); + auto cell_act = ctx.Attr("cell_activation"); + auto cand_act = ctx.Attr("candidate_activation"); auto batch_starts = batch_gate->lod()[0]; size_t num_batch = batch_starts.size() - 1; @@ -250,15 +296,15 @@ class LSTMGradKernel : public framework::OpKernel { lstm_grad.gateGrad = gate_g.data(); lstm_grad.outputGrad = out_g.data(); - if (n) { + if (n > 0) { int bstart_pre = static_cast(batch_starts[n - 1]); Tensor cell_pre = batch_cell.Slice(bstart_pre, bstart); Tensor cell_pre_g = batch_cell_g.Slice(bstart_pre, bstart); lstm_value.prevStateValue = cell_pre.data(); lstm_grad.prevStateGrad = cell_pre_g.data(); } else { - lstm_value.prevStateValue = nullptr; - lstm_grad.prevStateGrad = nullptr; + lstm_value.prevStateValue = c0 ? ordered_c0.data() : nullptr; + lstm_grad.prevStateGrad = c0_g ? ordered_c0_g.data() : nullptr; } int cur_batch_size = bend - bstart; @@ -266,7 +312,7 @@ class LSTMGradKernel : public framework::OpKernel { device_ctx, lstm_value, lstm_grad, frame_size, cur_batch_size, gate_act, cell_act, cand_act); - if (n != 0) { + if (n > 0) { int pre_h_start = static_cast(batch_starts[n - 1]); int pre_h_end = pre_h_start + cur_batch_size; auto pre_hidden_g = batch_hidden_g.Slice(pre_h_start, pre_h_end); @@ -280,6 +326,19 @@ class LSTMGradKernel : public framework::OpKernel { static_cast(1.0), weight_g, static_cast(1.0)); } + } else { + if (h0 && weight_g) { + ReorderInitState(device_ctx, *h0, order, &ordered_h0, true); + math::matmul(device_ctx, ordered_h0, true, gate_g, false, + static_cast(1.0), weight_g, + static_cast(1.0)); + } + if (h0 && h0_g) { + ordered_h0_g.mutable_data(h0_g->dims(), ctx.GetPlace()); + math::matmul(device_ctx, gate_g, false, *weight, true, + static_cast(1.0), &ordered_h0_g, + static_cast(0.0)); + } } } @@ -302,6 +361,13 @@ class LSTMGradKernel : public framework::OpKernel { math::gemv(device_ctx, true, m, n, 1., batch_gate_g.data(), ones.data(), 0., bias_g->data()); } + + if (h0 && h0_g) { + ReorderInitState(device_ctx, ordered_h0_g, order, h0_g, false); + } + if (c0 && c0_g) { + ReorderInitState(device_ctx, ordered_c0_g, order, c0_g, false); + } } }; diff --git a/paddle/operators/lstm_unit_op.cc b/paddle/operators/lstm_unit_op.cc index f4519ec16f3f6..18b9cdf2a39e8 100644 --- a/paddle/operators/lstm_unit_op.cc +++ b/paddle/operators/lstm_unit_op.cc @@ -34,10 +34,10 @@ class LstmUnitOp : public framework::OperatorWithKernel { auto c_prev_dims = ctx->GetInputDim("C_prev"); PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank must be 2."); - PADDLE_ENFORCE(x_dims[0] == c_prev_dims[0], - "Batch size of inputs and states must be equal"); - PADDLE_ENFORCE(x_dims[1] == c_prev_dims[1] * 4, - "Dimension of FC should equal to prev state * 4"); + PADDLE_ENFORCE_EQ(x_dims[0], c_prev_dims[0], + "Batch size of inputs and states must be equal"); + PADDLE_ENFORCE_EQ(x_dims[1], c_prev_dims[1] * 4, + "Dimension of FC should equal to prev state * 4"); int b_size = c_prev_dims[0]; // batch size int s_dim = c_prev_dims[1]; // state dim diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt index 90bc9f4f922e7..ab7f23f570438 100644 --- a/paddle/operators/math/CMakeLists.txt +++ b/paddle/operators/math/CMakeLists.txt @@ -13,7 +13,7 @@ if(WITH_GPU) nv_library(context_project SRCS context_project.cc context_project.cu DEPS device_context) nv_library(sequence2batch SRCS sequence2batch.cc sequence2batch.cu DEPS device_context) nv_library(lstm_compute SRCS lstm_compute.cc lstm_compute.cu DEPS device_context activation_functions) - nv_library(gru_compute SRCS gru_compute.cc gru_compute.cu DEPS device_context activation_functions) + nv_library(gru_compute SRCS gru_compute.cc gru_compute.cu DEPS device_context activation_functions math_function) else() cc_library(math_function SRCS math_function.cc im2col.cc DEPS cblas device_context operator) cc_library(selected_rows_functor SRCS selected_rows_functor.cc DEPS selected_rows math_function) diff --git a/paddle/operators/math/detail/lstm_cpu_kernel.h b/paddle/operators/math/detail/lstm_cpu_kernel.h index f5b0dd85c9d63..fc3ad0ce58aa1 100644 --- a/paddle/operators/math/detail/lstm_cpu_kernel.h +++ b/paddle/operators/math/detail/lstm_cpu_kernel.h @@ -52,9 +52,9 @@ void naive_lstm_forward_one_sequence(Op op, LstmMetaValue value, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = value.checkIg[i]; - rCheckF = value.checkFg[i]; - rCheckO = value.checkOg[i]; + rCheckI = value.checkIg ? value.checkIg[i] : 0; + rCheckF = value.checkFg ? value.checkFg[i] : 0; + rCheckO = value.checkOg ? value.checkOg[i] : 0; if (value.prevStateValue) { rPrevState = value.prevStateValue[i]; @@ -114,9 +114,9 @@ void naive_lstm_backward_one_sequence(Op op, LstmMetaValue value, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = value.checkIg[i]; - rCheckF = value.checkFg[i]; - rCheckO = value.checkOg[i]; + rCheckI = value.checkIg ? value.checkIg[i] : 0; + rCheckF = value.checkFg ? value.checkFg[i] : 0; + rCheckO = value.checkOg ? value.checkOg[i] : 0; rState = value.stateValue[i]; rStateAtv = value.stateActiveValue[i]; rOutputGrad = grad.outputGrad[i]; @@ -155,9 +155,9 @@ void avx_lstm_forward_one_sequence(Op op, LstmMetaValue value, int frameSize, __m256 rValueIg; __m256 rValueFg; __m256 rValueOg; - __m256 rCheckI; - __m256 rCheckF; - __m256 rCheckO; + __m256 rCheckI = _mm256_set1_ps(0.0f); + __m256 rCheckF = _mm256_set1_ps(0.0f); + __m256 rCheckO = _mm256_set1_ps(0.0f); __m256 rState; __m256 rPrevState = _mm256_set1_ps(0.0f); __m256 rStateAtv; @@ -173,9 +173,11 @@ void avx_lstm_forward_one_sequence(Op op, LstmMetaValue value, int frameSize, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = ((__m256 *)value.checkIg)[i]; - rCheckF = ((__m256 *)value.checkFg)[i]; - rCheckO = ((__m256 *)value.checkOg)[i]; + if (value.checkIg) { + rCheckI = ((__m256 *)value.checkIg)[i]; + rCheckF = ((__m256 *)value.checkFg)[i]; + rCheckO = ((__m256 *)value.checkOg)[i]; + } if (value.prevStateValue) { rPrevState = ((__m256 *)value.prevStateValue)[i]; @@ -216,9 +218,9 @@ void avx_lstm_backward_one_sequence(Op op, LstmMetaValue value, __m256 rState; __m256 rStateAtv; __m256 rOutputGrad; - __m256 rCheckI; - __m256 rCheckF; - __m256 rCheckO; + __m256 rCheckI = _mm256_set1_ps(0.0f); + __m256 rCheckF = _mm256_set1_ps(0.0f); + __m256 rCheckO = _mm256_set1_ps(0.0f); __m256 rCheckIGrad; __m256 rCheckFGrad; __m256 rCheckOGrad; @@ -237,9 +239,11 @@ void avx_lstm_backward_one_sequence(Op op, LstmMetaValue value, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = ((__m256 *)value.checkIg)[i]; - rCheckF = ((__m256 *)value.checkFg)[i]; - rCheckO = ((__m256 *)value.checkOg)[i]; + if (value.checkIg) { + rCheckI = ((__m256 *)value.checkIg)[i]; + rCheckF = ((__m256 *)value.checkFg)[i]; + rCheckO = ((__m256 *)value.checkOg)[i]; + } rState = ((__m256 *)value.stateValue)[i]; rStateAtv = ((__m256 *)value.stateActiveValue)[i]; rOutputGrad = ((__m256 *)grad.outputGrad)[i]; diff --git a/paddle/operators/math/detail/lstm_gpu_kernel.h b/paddle/operators/math/detail/lstm_gpu_kernel.h index 8b46510db05fb..d138bbe411f69 100644 --- a/paddle/operators/math/detail/lstm_gpu_kernel.h +++ b/paddle/operators/math/detail/lstm_gpu_kernel.h @@ -55,9 +55,10 @@ __global__ void KeLstmForward(Op op, LstmMetaValue value, int frameSize, T rValueIg; T rValueFg; T rValueOg; - T rCheckI = value.checkIg[frameIdx]; - T rCheckF = value.checkFg[frameIdx]; - T rCheckO = value.checkOg[frameIdx]; + + T rCheckI = value.checkIg ? value.checkIg[frameIdx] : 0; + T rCheckF = value.checkFg ? value.checkFg[frameIdx] : 0; + T rCheckO = value.checkOg ? value.checkOg[frameIdx] : 0; rValueIn = value.gateValue[frameIdx]; rValueIg = value.gateValue[frameIdx + frameSize]; @@ -121,9 +122,10 @@ __global__ void KeLstmBackward(Op op, LstmMetaValue value, T rStateGrad; T rStateAtv; T rOutputGrad; - T rCheckI = value.checkIg[frameIdx]; - T rCheckF = value.checkFg[frameIdx]; - T rCheckO = value.checkOg[frameIdx]; + T rCheckI = value.checkIg ? value.checkIg[frameIdx] : 0; + T rCheckF = value.checkFg ? value.checkFg[frameIdx] : 0; + T rCheckO = value.checkOg ? value.checkOg[frameIdx] : 0; + T rCheckIGrad; T rCheckFGrad; T rCheckOGrad; diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index 2a9c09a0f16b7..1b0d4c8bdc683 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/math/math_function.h" +#include "paddle/framework/data_type.h" namespace paddle { namespace operators { @@ -233,6 +234,52 @@ void gemv(const platform::DeviceContext& context, template struct SetConstant; +struct TensorSetConstantCPU { + TensorSetConstantCPU(framework::Tensor* tensor, float value) + : tensor_(tensor), value_(value) {} + template + void operator()() const { + auto cpu = platform::CPUPlace(); + auto* begin = tensor_->mutable_data(cpu); + std::fill(begin, begin + tensor_->numel(), static_cast(value_)); + } + framework::Tensor* tensor_; + float value_; +}; + +template <> +void set_constant_with_place( + const platform::DeviceContext& context, framework::Tensor* tensor, + float value) { + framework::VisitDataType(framework::ToDataType(tensor->type()), + TensorSetConstantCPU(tensor, value)); +} + +struct TensorSetConstantWithPlace : public boost::static_visitor { + TensorSetConstantWithPlace(const platform::DeviceContext& context, + framework::Tensor* tensor, float value) + : context_(context), tensor_(tensor), value_(value) {} + + template + void operator()(Place place) const { + set_constant_with_place(context_, tensor_, value_); + } + + const platform::DeviceContext& context_; + framework::Tensor* tensor_; + float value_; +}; + +void set_constant(const platform::DeviceContext& context, + framework::Tensor* tensor, float value) { + TensorSetConstantWithPlace func(context, tensor, value); +#ifdef PADDLE_WITH_CUDA + tensor->place().apply_visitor(func); +#else + func(platform::CPUPlace()); +#endif +} + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index e6fd8bf235b85..817deec94314b 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include "paddle/framework/data_type.h" #include "paddle/operators/math/math_function.h" namespace paddle { @@ -232,6 +233,30 @@ void gemv(const platform::DeviceContext& context, template struct SetConstant; +struct TensorSetConstantGPU { + TensorSetConstantGPU(const platform::DeviceContext& context, + framework::Tensor* tensor, float value) + : context_(context), tensor_(tensor), value_(value) {} + + template + void operator()() const { + SetConstant functor; + functor(context_, tensor_, static_cast(value_)); + } + + const platform::DeviceContext& context_; + framework::Tensor* tensor_; + float value_; +}; + +template <> +void set_constant_with_place( + const platform::DeviceContext& context, framework::Tensor* tensor, + float value) { + framework::VisitDataType(framework::ToDataType(tensor->type()), + TensorSetConstantGPU(context, tensor, value)); +} + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function.h b/paddle/operators/math/math_function.h index 3bb5aa0332c7e..c2aaa1d7b7e92 100644 --- a/paddle/operators/math/math_function.h +++ b/paddle/operators/math/math_function.h @@ -19,11 +19,6 @@ limitations under the License. */ #include #endif -#ifdef PADDLE_USE_MKL -#include -#include -#endif - #ifdef PADDLE_USE_ATLAS extern "C" { #include @@ -108,6 +103,13 @@ struct SetConstant { } }; +template +void set_constant_with_place(const platform::DeviceContext& context, + framework::Tensor* tensor, float value); + +void set_constant(const platform::DeviceContext& context, + framework::Tensor* tensor, float value); + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc index 7d84ad9aadb28..983c9fdcffb0a 100644 --- a/paddle/operators/math/math_function_test.cc +++ b/paddle/operators/math/math_function_test.cc @@ -139,3 +139,15 @@ TEST(math_function, gemv) { GemvTest(12, 7, true); GemvTest(7, 9, true); } + +TEST(math_funciton, set_constant) { + paddle::framework::Tensor t; + t.Resize({10, 10}); + t.mutable_data(paddle::platform::CPUPlace()); + auto* ctx = new paddle::platform::CPUDeviceContext(); + paddle::operators::math::set_constant(*ctx, &t, 10); + for (int64_t i = 0; i < t.numel(); ++i) { + PADDLE_ENFORCE_EQ(10, t.data()[i]); + } + delete ctx; +} diff --git a/paddle/operators/math/sequence2batch.cc b/paddle/operators/math/sequence2batch.cc index 10c6e105b950b..5b3bde02fbf98 100644 --- a/paddle/operators/math/sequence2batch.cc +++ b/paddle/operators/math/sequence2batch.cc @@ -22,8 +22,8 @@ template class CopyMatrixRowsFunctor { public: void operator()(const platform::DeviceContext& context, - const framework::LoDTensor& src, const size_t* index, - framework::LoDTensor& dst, bool is_src_index) { + const framework::Tensor& src, const size_t* index, + framework::Tensor& dst, bool is_src_index) { auto src_dims = src.dims(); auto dst_dims = dst.dims(); PADDLE_ENFORCE_EQ(src_dims.size(), 2UL, diff --git a/paddle/operators/math/sequence2batch.cu b/paddle/operators/math/sequence2batch.cu index 4f34994678517..8d04653832d58 100644 --- a/paddle/operators/math/sequence2batch.cu +++ b/paddle/operators/math/sequence2batch.cu @@ -41,8 +41,8 @@ template class CopyMatrixRowsFunctor { public: void operator()(const platform::DeviceContext& context, - const framework::LoDTensor& src, const size_t* index, - framework::LoDTensor& dst, bool is_src_index) { + const framework::Tensor& src, const size_t* index, + framework::Tensor& dst, bool is_src_index) { auto src_dims = src.dims(); auto dst_dims = dst.dims(); PADDLE_ENFORCE_EQ(src_dims.size(), 2, diff --git a/paddle/operators/math/sequence2batch.h b/paddle/operators/math/sequence2batch.h index b1ba35a6d4a89..794c7d4397392 100644 --- a/paddle/operators/math/sequence2batch.h +++ b/paddle/operators/math/sequence2batch.h @@ -30,8 +30,8 @@ class CopyMatrixRowsFunctor { // copy the input src to the indexed rows of output dst. // The indexed rows are based on the input index. void operator()(const platform::DeviceContext& context, - const framework::LoDTensor& src, const size_t* index, - framework::LoDTensor& dst, bool is_src_index); + const framework::Tensor& src, const size_t* index, + framework::Tensor& dst, bool is_src_index); }; template @@ -57,7 +57,7 @@ class LoDTensor2BatchFunctor { bool is_reverse = false) const { if (!is_cal_batch_lod) { auto lods = batch.lod(); - PADDLE_ENFORCE_EQ(lods.size(), 2UL); + PADDLE_ENFORCE_GT(lods.size(), 2UL); PADDLE_ENFORCE_EQ(lods[1].size(), static_cast(lod_tensor.dims()[0])); CopyMatrixRowsFunctor to_batch; @@ -66,8 +66,8 @@ class LoDTensor2BatchFunctor { } auto lods = lod_tensor.lod(); - PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now."); auto lod = lods[0]; + PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now."); std::vector seq_info; for (size_t seq_id = 0; seq_id < lod.size() - 1; ++seq_id) { @@ -78,8 +78,7 @@ class LoDTensor2BatchFunctor { std::sort(seq_info.begin(), seq_info.end(), [](SeqInfo a, SeqInfo b) { return a.length > b.length; }); - // calculate the start position of each batch - // (numBatch equal the maxLength of sequences) + // Calculate the start position of each batch. // example: sequences = {s0, s1, s2} // s0: 0 0 0 0, s1: 1 1 1 1 1, s2: 2 2 2 // num_batch = 5, @@ -95,19 +94,25 @@ class LoDTensor2BatchFunctor { // 6, 2, 11, // 7, 3, // 8} - // The batch number represents batch size after rearranging the + // seq_order = {1, 0, 2}, the sort order. + // where 1 is the second sequence, + // 0 is the first sequence, + // 2 is the third sequence. + // The num_batch represents batch size after rearranging the // input LodTensor. It is also the maximum length of input sequence. paddle::framework::LoD batch_lods; batch_lods.emplace_back(std::vector{0}); batch_lods.emplace_back(std::vector{0}); + batch_lods.emplace_back(std::vector{0}); // batch_lods[0] is the start positions for batch LoDTensor int num_batch = seq_info[0].length; batch_lods[0].resize(static_cast(num_batch + 1)); // batch_lods[1] is the raw index in the input LoDTensor - auto dims = lod_tensor.dims(); - batch_lods[1].resize(static_cast(dims[0])); + batch_lods[1].resize(static_cast(lod_tensor.dims()[0])); + // batch_lods[2] is the sort order for the input LoDTensor. + batch_lods[2].resize(seq_info.size()); size_t* batch_starts = batch_lods[0].data(); size_t* seq2batch_idx = batch_lods[1].data(); @@ -127,6 +132,10 @@ class LoDTensor2BatchFunctor { } batch_starts[n + 1] = static_cast(batch_id); } + size_t* seq_order = batch_lods[2].data(); + for (size_t i = 0; i < seq_info.size(); ++i) { + seq_order[i] = seq_info[i].seq_idx; + } batch.set_lod(batch_lods); CopyMatrixRowsFunctor to_batch; @@ -141,8 +150,7 @@ class Batch2LoDTensorFunctor { const framework::LoDTensor& batch, framework::LoDTensor& lod_tensor) const { auto in_lod = batch.lod(); - PADDLE_ENFORCE_EQ(in_lod.size(), 2UL, - "The LoD size of input `batch` should be 2."); + PADDLE_ENFORCE_GT(in_lod.size(), 2UL); PADDLE_ENFORCE_EQ(in_lod[1].size(), static_cast(lod_tensor.dims()[0])); CopyMatrixRowsFunctor to_seq; diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index 5ce30740c90b5..4f565946d596b 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -74,11 +74,10 @@ Tensor CombineBatchAndN(const framework::ExecutionContext& context, Tensor output; auto in_dims = input.dims(); if (in_dims.size() == 3) { - output.Resize(in_dims); + output.Resize({in_dims[1], in_dims[0], in_dims[2]}); output.mutable_data(context.GetPlace()); EigenTranspose(context, input, output, {1, 0, 2}); - std::vector out_dims = {in_dims[1], in_dims[0] * in_dims[2]}; - output.Resize(make_ddim(out_dims)); + output.Resize({in_dims[1], in_dims[0] * in_dims[2]}); } else { output.ShareDataWith(input); } diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 78b4bbca84d46..dcc5b4286f4ac 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -51,6 +51,7 @@ class MeanGradOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + ctx->ShareLoD("X", framework::GradVarName("X")); } }; diff --git a/paddle/operators/merge_lod_tensor_op.cc b/paddle/operators/merge_lod_tensor_op.cc new file mode 100644 index 0000000000000..80460c476921b --- /dev/null +++ b/paddle/operators/merge_lod_tensor_op.cc @@ -0,0 +1,182 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/memory/memcpy.h" + +namespace paddle { +namespace operators { + +using LoD = framework::LoD; + +class MergeLoDTensorOp : public framework::OperatorBase { + public: + MergeLoDTensorOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &mask = scope.FindVar(Input("Mask"))->Get(); + auto &in_true = scope.FindVar(Input("InTrue"))->Get(); + auto &in_false = + scope.FindVar(Input("InFalse"))->Get(); + auto *out = + scope.FindVar(Output("Out"))->GetMutable(); + auto level = static_cast(Attr("level")); + + auto &mask_dim = mask.dims(); + + std::unique_ptr cpu_mask{new framework::LoDTensor()}; + if (platform::is_cpu_place(mask.place())) { + cpu_mask->ShareDataWith(mask); + } else if (platform::is_gpu_place(mask.place())) { +#ifdef PADDLE_WITH_CUDA + cpu_mask->CopyFrom(mask, platform::CPUPlace(), dev_ctx); +#else + PADDLE_THROW("Not supported GPU, Please compile WITH_GPU option"); +#endif + } + auto *mask_data = cpu_mask->data(); + + int rank = in_true.dims().size(); + platform::Place place = in_true.place(); + std::type_index data_type = in_true.type(); + framework::DDim in_true_dims = + framework::slice_ddim(in_true.dims(), 1, rank); + + int64_t batch_size = in_true.dims()[0] + in_false.dims()[0]; + + auto in_true_dim_vec = framework::vectorize(in_true_dims); + in_true_dim_vec.insert(in_true_dim_vec.begin(), batch_size); + + framework::DDim out_dims = framework::make_ddim(in_true_dim_vec); + out->Resize(out_dims); + out->mutable_data(place, data_type); + + auto *out_lod = out->mutable_lod(); + out_lod->clear(); + size_t out_offset = 0; + + // Build LoDTensor `out` + + size_t in_true_idx = 0; + size_t in_false_idx = 0; + for (size_t i = 0; i < static_cast(mask_dim[0]); i++) { + const framework::LoDTensor *input = nullptr; + size_t *in_idx = nullptr; + if (static_cast(mask_data[i]) == 0) { + input = &in_false; + in_idx = &in_false_idx; + } else { + input = &in_true; + in_idx = &in_true_idx; + } + auto lod_and_offset = framework::GetSubLoDAndAbsoluteOffset( + input->lod(), *in_idx, (*in_idx) + 1, 0); + auto &lod_length = lod_and_offset.first; + + framework::AppendLoD(out_lod, lod_length); + + size_t start_offset = lod_and_offset.second.first; + size_t end_offset = lod_and_offset.second.second; + + PADDLE_ENFORCE_GE(end_offset, start_offset); + size_t len = end_offset - start_offset; + if (len == 0) { + continue; + } + out->Slice(out_offset, out_offset + len) + .CopyFrom(input->Slice(start_offset, end_offset), place, dev_ctx); + out_offset += len; + (*in_idx) += 1; + } + + for (size_t i = 0; i < level; i++) { + out_lod->insert(out_lod->begin(), x.lod()[i]); + } + } +}; + +class MergeLoDTensorOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + MergeLoDTensorOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input LoDTensor, contains complete lod information to " + "construct the output"); + AddInput("Mask", "A bool column vector which mask the input"); + AddInput("InTrue", "The True branch to be merged"); + AddInput("InFalse", "The False branch to be merged"); + AddOutput("Out", "The merged output LoDTensor"); + AddAttr("level", "(int) the specific lod level to rank.") + .SetDefault(0) + .EqualGreaterThan(0); + AddComment( + R"DOC( + Merge True and False branches of LoDTensor into a single Output, + with a mask at certain lod level. X is used to obtain complete + lod information. Please refer to SplitLoDTensorOp.)DOC"); + } +}; + +class MergeLoDTensorInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), + "MergeLoDTensorOp must has input X."); + PADDLE_ENFORCE(context->HasInput("Mask"), + "MergeLoDTensorOp must has input Mask."); + PADDLE_ENFORCE(context->HasInput("InTrue"), + "MergeLoDTensorOp must has input InTrue."); + PADDLE_ENFORCE(context->HasInput("InFalse"), + "MergeLoDTensorOp must has input InFalse."); + PADDLE_ENFORCE(context->HasOutput("Out"), + "MergeLoDTensorOp must has output Out"); + + auto mask_dim = context->GetInputDim("Mask"); + PADDLE_ENFORCE_EQ(mask_dim.size(), 2); + PADDLE_ENFORCE_EQ(mask_dim[1], 1); + + context->SetOutputDim("Out", context->GetInputDim("InTrue")); + } +}; + +class MergeLoDTensorGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("split_lod_tensor"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetInput("Mask", Input("Mask")); + grad_op->SetOutput("OutTrue", InputGrad("InTrue")); + grad_op->SetOutput("OutFalse", InputGrad("InFalse")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(merge_lod_tensor, ops::MergeLoDTensorOp, + ops::MergeLoDTensorOpProtoMaker, + ops::MergeLoDTensorInferShape, ops::MergeLoDTensorGradMaker); diff --git a/paddle/operators/momentum_op.cc b/paddle/operators/momentum_op.cc index e8ce16f4cfcf8..19954006195c1 100644 --- a/paddle/operators/momentum_op.cc +++ b/paddle/operators/momentum_op.cc @@ -75,7 +75,7 @@ class MomentumOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("VelocityOut", "(Tensor) Output updated velocity"); AddAttr("mu", "(float) Momentum coefficient"); - AddAttr("useNesterov", + AddAttr("use_nesterov", "(bool, default false) " "Use Nesterov Momentum") .SetDefault(false); diff --git a/paddle/operators/momentum_op.h b/paddle/operators/momentum_op.h index e6d6d1da3df9f..8f7f5eb5c21c0 100644 --- a/paddle/operators/momentum_op.h +++ b/paddle/operators/momentum_op.h @@ -34,7 +34,7 @@ class MomentumOpKernel : public framework::OpKernel { velocity_out->mutable_data(ctx.GetPlace()); float mu = ctx.Attr("mu"); - bool use_nesterov = ctx.Attr("useNesterov"); + bool use_nesterov = ctx.Attr("use_nesterov"); auto p_out = framework::EigenVector::Flatten(*param_out); auto v_out = framework::EigenVector::Flatten(*velocity_out); diff --git a/paddle/operators/mul_op.cu b/paddle/operators/mul_op.cu index a81444dbe63ed..66dc3d6d106a1 100644 --- a/paddle/operators/mul_op.cu +++ b/paddle/operators/mul_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/operators/mul_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/mul_op.h b/paddle/operators/mul_op.h index bd1bdb4f81b88..0eb9df41e9415 100644 --- a/paddle/operators/mul_op.h +++ b/paddle/operators/mul_op.h @@ -16,16 +16,12 @@ #include "paddle/operators/math/math_function.h" -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" namespace paddle { namespace operators { using Tensor = framework::Tensor; -template -using EigenMatrix = framework::EigenMatrix; template class MulKernel : public framework::OpKernel { diff --git a/paddle/operators/nccl/nccl_gpu_common.h b/paddle/operators/nccl/nccl_gpu_common.h index 5858cd4839d36..48e322f99398a 100644 --- a/paddle/operators/nccl/nccl_gpu_common.h +++ b/paddle/operators/nccl/nccl_gpu_common.h @@ -35,6 +35,7 @@ constexpr int kInvalidGPUId = -1; struct Communicator { std::vector comms_; std::unordered_map comm_id_map_; + bool inited_; Communicator() {} @@ -42,17 +43,21 @@ struct Communicator { void InitAll(const std::vector& gpus) { comms_.resize(gpus.size()); + inited_ = false; for (size_t i = 0; i < gpus.size(); ++i) { comm_id_map_[gpus[i]] = i; } PADDLE_ENFORCE( dynload::ncclCommInitAll(comms_.data(), gpus.size(), gpus.data())); + inited_ = true; } ~Communicator() { - for (size_t i = 0; i < comms_.size(); ++i) { - // FIXME(dzh) : PADDLE_ENFORCE return void - dynload::ncclCommDestroy(comms_[i]); + if (inited_) { + for (size_t i = 0; i < comms_.size(); ++i) { + // FIXME(dzh) : PADDLE_ENFORCE return void + dynload::ncclCommDestroy(comms_[i]); + } } } diff --git a/paddle/operators/nccl_op_test.cu b/paddle/operators/nccl_op_test.cu index e5927d56ae7cf..56ba57854955c 100644 --- a/paddle/operators/nccl_op_test.cu +++ b/paddle/operators/nccl_op_test.cu @@ -26,7 +26,6 @@ #include "paddle/framework/op_registry.h" #include "paddle/framework/program_desc.h" #include "paddle/framework/var_desc.h" -#include "paddle/operators/math/math_function.h" #include "paddle/operators/nccl/nccl_gpu_common.h" #include "paddle/platform/device_context.h" #include "paddle/platform/enforce.h" diff --git a/paddle/operators/pool_cudnn_op.cu b/paddle/operators/pool_cudnn_op.cu index 8d0741dccc1fd..8711567b95fea 100644 --- a/paddle/operators/pool_cudnn_op.cu +++ b/paddle/operators/pool_cudnn_op.cu @@ -37,11 +37,11 @@ class PoolCudnnOpKernel : public framework::OpKernel { const T *input_data = input->data(); T *output_data = output->mutable_data(ctx.GetPlace()); - std::string pooling_type = ctx.Attr("poolingType"); + std::string pooling_type = ctx.Attr("pooling_type"); std::vector ksize = ctx.Attr>("ksize"); std::vector strides = ctx.Attr>("strides"); std::vector paddings = ctx.Attr>("paddings"); - if (ctx.Attr("globalPooling")) { + if (ctx.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(input->dims()[i + 2]); @@ -92,12 +92,12 @@ class PoolCudnnGradOpKernel : public framework::OpKernel { ctx.Input(framework::GradVarName("Out")); Tensor *input_grad = ctx.Output(framework::GradVarName("X")); - std::string pooling_type = ctx.Attr("poolingType"); + std::string pooling_type = ctx.Attr("pooling_type"); std::vector ksize = ctx.Attr>("ksize"); std::vector strides = ctx.Attr>("strides"); std::vector paddings = ctx.Attr>("paddings"); - if (ctx.Attr("globalPooling")) { + if (ctx.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(input->dims()[i + 2]); diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index f58aab7338669..f3963b1995ef8 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -29,7 +29,7 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { auto in_x_dims = ctx->GetInputDim("X"); - std::string pooling_type = ctx->Attrs().Get("poolingType"); + std::string pooling_type = ctx->Attrs().Get("pooling_type"); std::vector ksize = ctx->Attrs().Get>("ksize"); std::vector strides = ctx->Attrs().Get>("strides"); std::vector paddings = ctx->Attrs().Get>("paddings"); @@ -37,7 +37,7 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5, "Pooling intput should be 4-D or 5-D tensor."); - if (ctx->Attrs().Get("globalPooling")) { + if (ctx->Attrs().Get("global_pooling")) { ksize.resize(static_cast(in_x_dims.size()) - 2); for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; @@ -83,20 +83,20 @@ Pool2dOpMaker::Pool2dOpMaker(framework::OpProto *proto, "H is the height of the feature, " "and W is the width of the feature."); - AddAttr("poolingType", + AddAttr("pooling_type", "(string), pooling type, can be \"max\" for max-pooling " "and \"avg\" for average-pooling.") .InEnum({"max", "avg"}); AddAttr>("ksize", "(vector) The pooling window " "size(height, width) of the pooling operator. " - "If globalPooling = true, ksize and paddings will " + "If global_pooling = true, ksize and paddings will " "be ignored."); // TODO(Chengduo): Add checker. // (Currently, // TypedAttrChecker don't support vector type.) - AddAttr("globalPooling", + AddAttr("global_pooling", "(bool, default false) Whether to use the global pooling. " - "If globalPooling = true, ksize and paddings will be ignored.") + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault(false); AddAttr>("strides", "(vector, default {1, 1}), strides(height, " @@ -107,7 +107,7 @@ Pool2dOpMaker::Pool2dOpMaker(framework::OpProto *proto, "paddings", "(vector, defalut {0,0}), paddings(height, width) of pooling " "operator." - "If globalPooling = true, paddings and ksize will be ignored.") + "If global_pooling = true, paddings and ksize will be ignored.") .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) @@ -115,7 +115,7 @@ Pool2dOpMaker::Pool2dOpMaker(framework::OpProto *proto, Pool2d Operator. The pooling2d operation calculates the output based on -the input, poolingType and ksize, strides, paddings parameters. +the input, pooling_type and ksize, strides, paddings parameters. Input(X) and output(Out) are in NCHW format, where N is batch size, C is the number of channels, H is the height of the feature, and W is the width of the feature. Parameters(ksize, strides, paddings) are two elements. @@ -152,7 +152,7 @@ Pool3dOpMaker::Pool3dOpMaker(framework::OpProto *proto, "the number of channels, and D, H and W is the depth, height and " "width of the feature, respectively."); - AddAttr("poolingType", + AddAttr("pooling_type", "(string) Pooling type, can be \"max\" for max-pooling " "and \"avg\" for average-pooling.") .InEnum({"max", "avg"}); @@ -160,13 +160,14 @@ Pool3dOpMaker::Pool3dOpMaker(framework::OpProto *proto, "ksize", "(vector) The pooling window size(depth, height, " "width) of pooling operator. " - "If globalPooling = true, ksize and paddings will " + "If global_pooling = true, ksize and paddings will " "be ignored."); // TODO(Chengduo): Add checker. // (Currently, // TypedAttrChecker don't support vector type.) - AddAttr("globalPooling", - "(bool, default false) Whether to use the global pooling. " - "If globalPooling = true, ksize and paddings wille be ignored.") + AddAttr( + "global_pooling", + "(bool, default false) Whether to use the global pooling. " + "If global_pooling = true, ksize and paddings wille be ignored.") .SetDefault(false); AddAttr>( "strides", @@ -178,7 +179,7 @@ Pool3dOpMaker::Pool3dOpMaker(framework::OpProto *proto, "paddings", "(vector, defalut {0,0,0}), paddings(depth, height, " "width) of pooling operator. " - "If globalPooling = true, ksize and paddings will be ignored.") + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) @@ -186,7 +187,7 @@ Pool3dOpMaker::Pool3dOpMaker(framework::OpProto *proto, Pool3d Operator. The pooling3d operation calculates the output based on -the input, poolingType, ksize, strides, and paddings parameters. +the input, pooling_type, ksize, strides, and paddings parameters. Input(X) and output(Out) are in NCDHW format, where N is batch size, C is the number of channels, and D, H and W are the depth, height and width of the feature, respectively. Parameters(ksize, strides, paddings) diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index d9d445f6a6257..4da1941ab5414 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -57,11 +57,11 @@ class PoolKernel : public framework::OpKernel { const Tensor* in_x = context.Input("X"); Tensor* out = context.Output("Out"); - std::string pooling_type = context.Attr("poolingType"); + std::string pooling_type = context.Attr("pooling_type"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x->dims()[i + 2]); @@ -119,12 +119,12 @@ class PoolGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out")); Tensor* in_x_grad = context.Output(framework::GradVarName("X")); - std::string pooling_type = context.Attr("poolingType"); + std::string pooling_type = context.Attr("pooling_type"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x->dims()[i + 2]); diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc index a31b3fcb7083b..1df36e965abab 100644 --- a/paddle/operators/pool_with_index_op.cc +++ b/paddle/operators/pool_with_index_op.cc @@ -44,7 +44,7 @@ class MaxPoolWithIndexOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5, "Pooling intput should be 4-D or 5-D tensor."); - if (ctx->Attrs().Get("globalPooling")) { + if (ctx->Attrs().Get("global_pooling")) { ksize.resize(static_cast(in_x_dims.size()) - 2); for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; @@ -110,14 +110,14 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr>("ksize", "(vector) The pooling window size(height, " "width) of pooling operator. " - "If globalPooling = true, ksize and paddings " + "If global_pooling = true, ksize and paddings " "will be ignored."); // TODO(Chengduo): Add // checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr( - "globalPooling", + "global_pooling", "(bool, default false) Whether to use the global pooling. " - "If globalPooling = true, ksize and paddings will be ignored.") + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault(false); AddAttr>("strides", "(vector, default {1, 1}), strides(height, " @@ -128,7 +128,7 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { "paddings", "(vector, defalut {0, 0}), paddings(height, width) of pooling " "operator. " - "If globalPooling = true, paddings and will be ignored.") + "If global_pooling = true, paddings and will be ignored.") .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) @@ -188,14 +188,14 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr>("ksize", "(vector) The pooling window size(depth, " "height, width) of pooling operator. " - "If globalPooling = true, ksize and paddings " + "If global_pooling = true, ksize and paddings " "will be ignored."); // TODO(Chengduo): Add // checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr( - "globalPooling", + "global_pooling", "(bool, default false) Whether to use the global pooling. " - "If globalPooling = true, ksize and paddings will be ignored.") + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault(false); AddAttr>("strides", "(vector, default {1,1,1}), strides(depth, " @@ -206,7 +206,7 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { "paddings", "(vector, defalut {0,0,0}), paddings(depth, " "height, width) of pooling operator. " - "If globalPooling = true, paddings and ksize will be ignored.") + "If global_pooling = true, paddings and ksize will be ignored.") .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) diff --git a/paddle/operators/pool_with_index_op.h b/paddle/operators/pool_with_index_op.h index 48627740435b7..ea37de84abeb5 100644 --- a/paddle/operators/pool_with_index_op.h +++ b/paddle/operators/pool_with_index_op.h @@ -35,7 +35,7 @@ class MaxPoolWithIndexKernel : public framework::OpKernel { std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x->dims()[i + 2]); @@ -72,7 +72,7 @@ class MaxPoolWithIndexGradKernel : public framework::OpKernel { std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x_grad->dims()[i + 2]); diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index b0e87b7059eab..0075ccd24271b 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -387,8 +387,8 @@ class RecurrentGradOp : public RecurrentBase { auto &p_names = Inputs(kParameters); PADDLE_ENFORCE_EQ(pg_names.size(), p_names.size()); - for (size_t prog_id = 0; prog_id < pg_names.size(); ++prog_id) { - auto inside_grad_name = framework::GradVarName(p_names[prog_id]); + for (size_t param_id = 0; param_id < pg_names.size(); ++param_id) { + auto inside_grad_name = framework::GradVarName(p_names[param_id]); // If does not compute gradient of that variable inside rnn, just // continue @@ -406,27 +406,19 @@ class RecurrentGradOp : public RecurrentBase { attrs["value"] = 0.0f; auto zero_op = framework::OpRegistry::CreateOp( - "fill_constant", {}, {{"Out", {pg_names[prog_id]}}}, attrs); + "fill_constant", {}, {{"Out", {pg_names[param_id]}}}, attrs); zero_op->Run(scope, dev_ctx); } + auto new_inside_name = cur_scope.Rename(inside_grad_name); // sum gradient - auto *outside_var = scope.FindVar(pg_names[prog_id]); - PADDLE_ENFORCE(outside_var != nullptr); - auto &outside_tensor = - *outside_var->GetMutable(); - - std::string result_var_name; - auto *local_result_var = cur_scope.Var(&result_var_name); - auto &local_result_tensor = - *local_result_var->GetMutable(); - - local_result_tensor.ShareDataWith(outside_tensor); auto sum_op = framework::OpRegistry::CreateOp( - "sum", {{"X", {result_var_name, inside_grad_name}}}, - {{"Out", {result_var_name}}}, {}); + "sum", {{"X", {pg_names[param_id], new_inside_name}}}, + {{"Out", {pg_names[param_id]}}}, {}); sum_op->Run(cur_scope, dev_ctx); + + cur_scope.Rename(new_inside_name, inside_grad_name); } } VLOG(5) << "Accumulate Parameter finished "; diff --git a/paddle/operators/sequence_concat_op.cc b/paddle/operators/sequence_concat_op.cc index 64097ef2525d7..d1de0b444712a 100644 --- a/paddle/operators/sequence_concat_op.cc +++ b/paddle/operators/sequence_concat_op.cc @@ -47,7 +47,7 @@ class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", - "(vector) Input is a vector of LoDTensor, " + "(LodTensorArray) Input is a vector of LoDTensor, " "each of which is a variable-length sequence or nested sequence.") .AsDuplicable(); AddOutput("Out", @@ -68,38 +68,42 @@ class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker { "The level should be less than the level number of inputs.") .SetDefault(0); AddComment(R"DOC( -Sequence Concat Operator. - -The sequence_concat operator concatenates multiple LoDTensors. -It supports a sequence (LoD Tensor with level number is 1) +The sequence_concat operator concatenates multiple LoDTensors. +It only supports sequence (LoD Tensor with level number is 1) or a nested sequence (LoD tensor with level number is 2) as its input. -The following examples explain how the operator works: - Case1: If the axis is other than 0(here, axis is 1 and level is 1), - each input should have the same LoD information and the LoD + each input should have the same LoD information and the LoD information of the output keeps the same as the input. - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,2,4}, {0,1,2,3,4}}; Dims(x1) = (4,4,4) - LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4) + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,2,4}, {0,1,2,3,4}}; Dims(x1) = (4,4,4) + LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4) - Case2: - If the axis is 0(here, leve is 0), the inputs are concatenated along + If the axis is 0(here, leve is 0), the inputs are concatenated along time steps, the LoD information of the output need to re-compute. + The LoD information of level-1 should be same. - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,3,5}, {0,1,2,3,5}}; Dims(x1) = (5,3,4) - LoD(Out) = {{0,5,9}, {0,1,2,3,4,5,6,7,9}}; Dims(Out) = (9,3,4) + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,2,4}, {0,1,3,5,7}}; Dims(x1) = (7,3,4) + LoD(Out) = {{0,2,4}, {0,2,5,8,11}}; Dims(Out) = (11,3,4) - Case3: If the axis is 0(here, level is 1). - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,3,5}, {0,1,3,4,5}}; Dims(x1) = (5,3,4) - LoD(Out) = {{0,5,9}, {0,2,5,7,9}}; Dims(Out) = (9,3,4) + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,3,4}, {0,1,3,5,7}}; Dims(x1) = (7,3,4) + LoD(Out) = {{0,5,8}, {0,1,2,3,5,7,8,9,11}}; Dims(Out) = (11,3,4) -NOTE: The levels of all the inputs should be the same. +- Case4: + If the LoD number is 1, axis is 0, level is 0 + LoD(x0) = {{0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,1,3,5,7}}; Dims(x1) = (7,3,4) + LoD(Out) = {{0,2,5,8,11}}; Dims(Out) = (11,3,4) + +NOTE: The levels of all the inputs should be the same. )DOC"); } }; diff --git a/paddle/operators/sequence_concat_op.cu b/paddle/operators/sequence_concat_op.cu index 8dc4764785871..9ca99c2258f54 100644 --- a/paddle/operators/sequence_concat_op.cu +++ b/paddle/operators/sequence_concat_op.cu @@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU - #include "paddle/operators/sequence_concat_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/sequence_concat_op.h b/paddle/operators/sequence_concat_op.h index 6adf96120c99f..09212070aa90b 100644 --- a/paddle/operators/sequence_concat_op.h +++ b/paddle/operators/sequence_concat_op.h @@ -24,28 +24,38 @@ using LoDTensor = framework::LoDTensor; using LoD = framework::LoD; template -LoD concatLoD(const std::vector ins, const size_t axis, - const size_t level) { +LoD ConcatLoD(const std::vector ins, const size_t level) { auto out_lod = ins[0]->lod(); + auto numLevels = ins[0]->NumLevels(); const size_t n = ins.size(); - if (axis == 0UL) { - for (size_t i = 1; i < n; ++i) { - for (size_t j = 0; j < ins[i]->lod()[0].size(); ++j) { - out_lod[0][j] += ins[i]->lod()[0][j]; - } + const size_t level_idx = ins[0]->NumLevels() - 1 - level; + for (size_t i = 1; i < n; ++i) { + for (size_t j = 0; j < ins[i]->lod()[level_idx].size(); ++j) { + out_lod[level_idx][j] += ins[i]->lod()[level_idx][j]; + } + } - if (ins[0]->NumLevels() == 2) { - for (size_t j = 1; j < ins[i]->lod()[1].size(); ++j) { - if (level == 0UL) { - out_lod[1].push_back(out_lod[1].back() + ins[i]->lod()[1][j] - - ins[i]->lod()[1][j - 1]); - } else if (level == 1UL) { - out_lod[1][j] += ins[1]->lod()[1][j]; - } + for (size_t i = level_idx; i < numLevels - 1; ++i) { + size_t lod_len = 1; + for (size_t j = 0; j < n; ++j) { + lod_len += ins[j]->lod()[i + 1].size() - 1; + } + out_lod[i + 1].clear(); + out_lod[i + 1].resize(lod_len); + + size_t idx = 1; + for (size_t j = 0; j < ins[0]->lod()[i].size() - 1; ++j) { + for (size_t k = 0; k < n; ++k) { + for (size_t m = ins[k]->lod()[i][j]; m < ins[k]->lod()[i][j + 1]; ++m) { + out_lod[i + 1][idx] = out_lod[i + 1][idx - 1] + + ins[k]->lod()[i + 1][m + 1] - + ins[k]->lod()[i + 1][m]; + idx++; } } } } + return out_lod; } @@ -82,18 +92,21 @@ class SequenceConcatOpKernel : public framework::OpKernel { "should be greater than the specify level"); out->mutable_data(ctx.GetPlace()); - auto out_lod = concatLoD(ins, axis, level); + auto out_lod = ins[0]->lod(); + if (axis == 0) { + out_lod = ConcatLoD(ins, level); + } out->set_lod(out_lod); - auto out_lod_level = out_lod[level]; + const size_t level_idx = out_lod.size() - level - 1; + auto out_lod_level = framework::ToAbsOffset(out_lod)[level_idx]; for (size_t i = 0; i < out_lod_level.size() - 1; ++i) { Tensor out_t = out->Slice(static_cast(out_lod_level[i]), static_cast(out_lod_level[i + 1])); auto out_stride = framework::stride(out_t.dims()); size_t offset = 0; - for (size_t j = 0; j < n; ++j) { - auto in_lod_level = ins[j]->lod()[level]; + auto in_lod_level = framework::ToAbsOffset(ins[j]->lod())[level_idx]; auto in_stride = framework::stride(ins[j]->dims()); Tensor in_t = ins[j]->Slice(static_cast(in_lod_level[i]), static_cast(in_lod_level[i + 1])); @@ -124,9 +137,12 @@ class SequenceConcatGradOpKernel : public framework::OpKernel { x_grads[i]->set_lod(ins[i]->lod()); x_grads[i]->mutable_data(ctx.GetPlace()); } - - auto out_lod = concatLoD(ins, axis, level); - auto out_lod_level = out_lod[level]; + auto out_lod = ins[0]->lod(); + if (axis == 0UL) { + out_lod = ConcatLoD(ins, level); + } + const size_t level_idx = out_lod.size() - level - 1; + auto out_lod_level = framework::ToAbsOffset(out_lod)[level_idx]; for (size_t i = 0; i < out_lod_level.size() - 1; ++i) { Tensor out_grad_t = @@ -136,7 +152,8 @@ class SequenceConcatGradOpKernel : public framework::OpKernel { size_t offset = 0; for (size_t j = 0; j < n; ++j) { - auto x_grad_lod_level = x_grads[j]->lod()[level]; + auto x_grad_lod_level = + framework::ToAbsOffset(x_grads[j]->lod())[level_idx]; auto x_grad_stride = framework::stride(x_grads[j]->dims()); Tensor x_grad_t = x_grads[j]->Slice(static_cast(x_grad_lod_level[i]), diff --git a/paddle/operators/sequence_pool_op.h b/paddle/operators/sequence_pool_op.h index 2b8a25c2414c2..7f136d8cf0e1e 100644 --- a/paddle/operators/sequence_pool_op.h +++ b/paddle/operators/sequence_pool_op.h @@ -126,6 +126,7 @@ class SequencePoolGradKernel : public framework::OpKernel { int64_t h = static_cast(lod[i + 1] - lod[i]); auto in_g_e = EigenMatrix::From(in_g_t, {h, w}); auto out_g_e = EigenMatrix::From(out_g_t, {1, w}); + auto out_g_e_v = EigenVector::Flatten(out_g_t); Eigen::DSizes bcast(h, 1); if (pooltype == "AVERAGE") { @@ -136,9 +137,9 @@ class SequencePoolGradKernel : public framework::OpKernel { in_g_e.device(place) = (out_g_e / std::sqrt(static_cast(h))).broadcast(bcast); } else if (pooltype == "LAST") { - in_g_e.chip(h - 1, 0).device(place) = out_g_e; + in_g_e.chip(h - 1, 0).device(place) = out_g_e_v; } else if (pooltype == "FIRST") { - in_g_e.chip(0, 0).device(place) = out_g_e; + in_g_e.chip(0, 0).device(place) = out_g_e_v; } else { PADDLE_THROW("unsupported pooling pooltype"); } diff --git a/paddle/operators/sequence_softmax_op.cu b/paddle/operators/sequence_softmax_op.cu index f2a1e3d5e31ef..7023795a3b577 100644 --- a/paddle/operators/sequence_softmax_op.cu +++ b/paddle/operators/sequence_softmax_op.cu @@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU - #include "paddle/operators/sequence_softmax_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/sequence_softmax_op.h b/paddle/operators/sequence_softmax_op.h index 3eb1e2844dff6..1b68dd0662ddf 100644 --- a/paddle/operators/sequence_softmax_op.h +++ b/paddle/operators/sequence_softmax_op.h @@ -14,7 +14,6 @@ limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/math/softmax.h" diff --git a/paddle/operators/shrink_rnn_memory_op.cc b/paddle/operators/shrink_rnn_memory_op.cc new file mode 100644 index 0000000000000..65bccc0c81d0a --- /dev/null +++ b/paddle/operators/shrink_rnn_memory_op.cc @@ -0,0 +1,149 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "paddle/framework/lod_rank_table.h" +#include "paddle/operators/array_operator.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +class ShrinkRNNMemoryOp : public ArrayOp { + public: + ShrinkRNNMemoryOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ArrayOp(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *x_var = scope.FindVar(Input("X")); + PADDLE_ENFORCE(x_var != nullptr, "Input X must be set"); + auto &x_tensor = x_var->Get(); + size_t offset = this->GetOffset(scope, dev_ctx); + auto *rank_table_var = scope.FindVar(Input("RankTable")); + PADDLE_ENFORCE(rank_table_var != nullptr, "RankTable must be set"); + auto &rank_table = rank_table_var->Get(); + + auto &rank_items = rank_table.items(); + int dst_num_rows = + std::lower_bound(rank_items.begin(), rank_items.end(), offset, + [](const framework::LoDRankTable::TableItem &a, + size_t b) { return a.length > b; }) - + rank_items.begin(); + + auto *out_var = scope.FindVar(Output("Out")); + PADDLE_ENFORCE(out_var != nullptr, "Output Out must be set"); + auto &out_tensor = *out_var->GetMutable(); + if (dst_num_rows != 0) { + out_tensor.ShareDataWith(x_tensor.Slice(0, dst_num_rows)); + } + } +}; + +class ShrinkRNNMemoryOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ShrinkRNNMemoryOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", ""); + AddInput("RankTable", ""); + AddInput("I", ""); + AddOutput("Out", ""); + AddComment(""); + } +}; + +class ShrinkRNNMemoryInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X")); + PADDLE_ENFORCE(context->HasInput("I")); + PADDLE_ENFORCE(context->HasInput("RankTable")); + context->SetOutputDim("Out", context->GetInputDim("X")); + } +}; + +class ShrinkRNNMemoryGradOp : public ArrayOp { + public: + ShrinkRNNMemoryGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ArrayOp(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *dout_var = scope.FindVar(Input(framework::GradVarName("Out"))); + auto *dx_var = scope.FindVar(Output(framework::GradVarName("X"))); + PADDLE_ENFORCE(dx_var != nullptr, "Input Gradient should not be nullptr"); + auto *x_var = scope.FindVar(Input("X")); + PADDLE_ENFORCE(x_var != nullptr); + + auto &x_tensor = x_var->Get(); + auto &dx_tensor = *dx_var->GetMutable(); + dx_tensor.Resize(x_tensor.dims()); + dx_tensor.mutable_data(x_tensor.place(), x_tensor.type()); + + if (dout_var == nullptr) { // dx_tensor fill zero + math::set_constant(dev_ctx, &dx_tensor, 0.0f); + } else { + auto &dout_tensor = dout_var->Get(); + auto height = dout_tensor.dims()[0]; + dx_tensor.Slice(0, static_cast(height)) + .CopyFrom(dout_tensor, dout_tensor.place(), dev_ctx); + if (dx_tensor.dims()[0] < height) { + auto rest_tensor = dx_tensor.Slice( + static_cast(height), static_cast(dout_tensor.dims()[0])); + math::set_constant(dev_ctx, &rest_tensor, 0.0f); + } + } + } +}; + +class ShrinkRNNMemoryGradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X")); + PADDLE_ENFORCE(context->HasOutput(framework::GradVarName("X"))); + context->SetOutputDim(framework::GradVarName("X"), + context->GetInputDim("X")); + } +}; + +class ShrinkRNNGradOpMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *op = new framework::OpDescBind(); + op->SetType("shrink_rnn_memory_grad"); + op->SetInput("X", Input("X")); + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + op->SetAttrMap(Attrs()); + return std::unique_ptr(op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(shrink_rnn_memory, ops::ShrinkRNNMemoryOp, + ops::ShrinkRNNMemoryInferShape, + ops::ShrinkRNNMemoryOpProtoMaker, ops::ShrinkRNNGradOpMaker); +REGISTER_OPERATOR(shrink_rnn_memory_grad, ops::ShrinkRNNMemoryGradOp, + ops::ShrinkRNNMemoryGradInferShape); diff --git a/paddle/operators/softmax_op.cu b/paddle/operators/softmax_op.cu index 2e99a89699dbd..013ace19ae3d4 100644 --- a/paddle/operators/softmax_op.cu +++ b/paddle/operators/softmax_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/operators/softmax_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/softmax_op.h b/paddle/operators/softmax_op.h index 2c08853f4f615..44d1e63f1bb47 100644 --- a/paddle/operators/softmax_op.h +++ b/paddle/operators/softmax_op.h @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/math/softmax.h" @@ -21,9 +20,6 @@ namespace paddle { namespace operators { using Tensor = framework::Tensor; -template -using EigenMatrix = framework::EigenMatrix; template class SoftmaxKernel : public framework::OpKernel { diff --git a/paddle/operators/split_lod_tensor_op.cc b/paddle/operators/split_lod_tensor_op.cc new file mode 100644 index 0000000000000..db635f2ba0804 --- /dev/null +++ b/paddle/operators/split_lod_tensor_op.cc @@ -0,0 +1,186 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/memory/memcpy.h" + +namespace paddle { +namespace operators { + +struct CopyRange { + size_t begin; + size_t end; +}; + +using LoD = framework::LoD; + +class SplitLoDTensorOp : public framework::OperatorBase { + public: + SplitLoDTensorOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &mask = scope.FindVar(Input("Mask"))->Get(); + auto *out_true = + scope.FindVar(Output("OutTrue"))->GetMutable(); + auto *out_false = + scope.FindVar(Output("OutFalse"))->GetMutable(); + auto level = static_cast(Attr("level")); + auto &x_lod = x.lod(); + auto &mask_dim = mask.dims(); + + std::unique_ptr cpu_mask{new framework::LoDTensor()}; + if (platform::is_cpu_place(mask.place())) { + cpu_mask->ShareDataWith(mask); + } else if (platform::is_gpu_place(mask.place())) { +#ifdef PADDLE_WITH_CUDA + cpu_mask->CopyFrom(mask, platform::CPUPlace(), dev_ctx); +#else + PADDLE_THROW("Not supported GPU, Please compile WITH_GPU option"); +#endif + } + auto *mask_data = cpu_mask->data(); + + std::vector> copy_ranges(mask_dim[0]); + + // set out_true/out_false lod + for (size_t t = 0; t < 2; t++) { + LoD *lod = nullptr; + if (t == 0) { + lod = out_false->mutable_lod(); + } else { + lod = out_true->mutable_lod(); + } + lod->clear(); + for (size_t i = 0; i < static_cast(mask_dim[0]); i++) { + if (static_cast(mask_data[i]) == t) { + size_t start_idx = i; + auto lod_and_offset = framework::GetSubLoDAndAbsoluteOffset( + x_lod, start_idx, start_idx + 1, level); + + auto &lod_length = lod_and_offset.first; + framework::AppendLoD(lod, lod_length); + + size_t start_offset = lod_and_offset.second.first; + size_t end_offset = lod_and_offset.second.second; + copy_ranges[t].emplace_back(CopyRange{start_offset, end_offset}); + } + } + } + + for (size_t t = 0; t < 2; ++t) { + framework::LoDTensor *out; + if (t == 0) { + out = out_false; + } else { + out = out_true; + } + auto &ranges = copy_ranges[t]; + size_t height = std::accumulate( + ranges.begin(), ranges.end(), 0UL, + [](size_t a, const CopyRange &b) { return a + b.end - b.begin; }); + auto x_dim = x.dims(); + x_dim[0] = static_cast(height); + out->Resize(x_dim); + out->mutable_data(x.place(), x.type()); + size_t offset = 0; + for (auto &each_range : ranges) { + size_t len = each_range.end - each_range.begin; + if (len == 0) { + continue; + } + // out[offset: offset+len] = x[each_range.begin: each_range.end] + out->Slice(static_cast(offset), static_cast(offset + len)) + .CopyFrom(x.Slice(static_cast(each_range.begin), + static_cast(each_range.end)), + x.place(), dev_ctx); + offset += len; + } + } + } +}; + +class SplitLoDTensorOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + SplitLoDTensorOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The input LoDTensor"); + AddInput("Mask", "A bool column vector which mask the input"); + AddOutput("OutTrue", "True branch of input LoDTensor"); + AddOutput("OutFalse", "False branch of input LoDTensor"); + AddAttr("level", "(int) the specific lod level to split.") + .SetDefault(0) + .EqualGreaterThan(0); + AddComment( + R"DOC( + Split a LoDTensor with a Mask at certain level. The input LoDTensor + has 3 sequence at certain lod level. The Mask is a bool column vector, + such as [0, 1, 0] at the same level. The first and third sequence will + be send to False Output LoDTensor; whereas the second sequence will + be send to True Output LoDTensor. Please refer to MergeLoDTensorOp.)DOC"); + } +}; + +class SplitLoDTensorInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), + "SplitLoDTensorOp must has input X."); + PADDLE_ENFORCE(context->HasInput("Mask"), + "SplitLoDTensorOp must has input Mask."); + PADDLE_ENFORCE(context->HasOutput("OutTrue"), + "SplitLoDTensorOp must has output OutTrue."); + PADDLE_ENFORCE(context->HasOutput("OutFalse"), + "SplitLoDTensorOp must has output OutFalse."); + + auto mask_dim = context->GetInputDim("Mask"); + PADDLE_ENFORCE_EQ(mask_dim.size(), 2); + PADDLE_ENFORCE_EQ(mask_dim[1], 1); + + context->SetOutputDim("OutTrue", context->GetInputDim("X")); + context->SetOutputDim("OutFalse", context->GetInputDim("X")); + } +}; + +class SplitLoDTensorArrayGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("merge_lod_tensor"); + grad_op->SetInput("InTrue", OutputGrad("OutTrue")); + grad_op->SetInput("InFalse", OutputGrad("OutFalse")); + grad_op->SetInput("Mask", Input("Mask")); + grad_op->SetInput("X", Input("X")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(split_lod_tensor, ops::SplitLoDTensorOp, + ops::SplitLoDTensorOpProtoMaker, + ops::SplitLoDTensorInferShape, + ops::SplitLoDTensorArrayGradMaker); diff --git a/paddle/operators/squared_l2_norm_op.h b/paddle/operators/squared_l2_norm_op.h index c8d37ac40c153..48d7b1c2d5688 100644 --- a/paddle/operators/squared_l2_norm_op.h +++ b/paddle/operators/squared_l2_norm_op.h @@ -29,7 +29,7 @@ class SquaredL2NormKernel : public framework::OpKernel { Out->mutable_data(context.GetPlace()); auto x = framework::EigenVector::Flatten(*X); - auto out = framework::EigenVector::Flatten(*Out); + auto out = framework::EigenScalar::From(*Out); auto place = context.GetEigenDevice(); out.device(place) = x.square().sum(); diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 750f96296a841..57b99bdb3a935 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -99,11 +99,12 @@ class SumOpVarTypeInference : public framework::VarTypeInference { bool any_input_is_lod_tensor = std::any_of( inputs.begin(), inputs.end(), [block](const std::string& name) { - return block->Var(name)->GetType() == framework::VarDesc::LOD_TENSOR; + return block->FindRecursiveOrCreateVar(name)->GetType() == + framework::VarDesc::LOD_TENSOR; }); auto is_tensor_array = [block](const std::string& name) { - return block->Var(name)->GetType() == + return block->FindRecursiveOrCreateVar(name)->GetType() == framework::VarDesc::LOD_TENSOR_ARRAY; }; @@ -120,7 +121,7 @@ class SumOpVarTypeInference : public framework::VarTypeInference { } auto out_var_name = op_desc.Output("Out").front(); - block->Var(out_var_name)->SetType(var_type); + block->FindRecursiveOrCreateVar(out_var_name)->SetType(var_type); } }; diff --git a/paddle/operators/tensor_array_read_write_op.cc b/paddle/operators/tensor_array_read_write_op.cc index 50824032ca0e2..62e15604c47f2 100644 --- a/paddle/operators/tensor_array_read_write_op.cc +++ b/paddle/operators/tensor_array_read_write_op.cc @@ -11,48 +11,18 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/framework/lod_tensor_array.h" -#include "paddle/framework/op_registry.h" +#include "paddle/operators/array_operator.h" namespace paddle { namespace operators { -class ArrayOpBase : public framework::OperatorBase { - public: - ArrayOpBase(const std::string &type, const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) - : OperatorBase(type, inputs, outputs, attrs) {} - void Run(const framework::Scope &scope, - const platform::DeviceContext &dev_ctx) const override {} - - protected: - size_t GetOffset(const framework::Scope &scope, - const platform::DeviceContext &dev_ctx) const { - auto *i = scope.FindVar(Input("I")); - PADDLE_ENFORCE(i != nullptr, "I must be set"); - auto &i_tensor = i->Get(); - PADDLE_ENFORCE_EQ(i_tensor.numel(), 1); - size_t offset; - if (platform::is_gpu_place(i_tensor.place())) { - // FIXME: Avoid copy from GPU to CPU - framework::Tensor t; - t.CopyFrom(i_tensor, platform::CPUPlace(), dev_ctx); - dev_ctx.Wait(); - offset = static_cast(*t.data()); - } else { - offset = static_cast(*i_tensor.data()); - } - return offset; - } -}; -class WriteToArrayOp : public ArrayOpBase { +class WriteToArrayOp : public ArrayOp { public: WriteToArrayOp(const std::string &type, const framework::VariableNameMap &inputs, const framework::VariableNameMap &outputs, const framework::AttributeMap &attrs) - : ArrayOpBase(type, inputs, outputs, attrs) {} + : ArrayOp(type, inputs, outputs, attrs) {} void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { @@ -117,18 +87,19 @@ class WriteToArrayInferVarType : public framework::VarTypeInference { framework::BlockDescBind *block) const override { for (auto &out_var : op_desc.OutputArgumentNames()) { VLOG(10) << "Set Variable " << out_var << " as LOD_TENSOR_ARRAY"; - block->Var(out_var)->SetType(framework::VarDesc::LOD_TENSOR_ARRAY); + block->FindRecursiveOrCreateVar(out_var)->SetType( + framework::VarDesc::LOD_TENSOR_ARRAY); } } }; -class ReadFromArrayOp : public ArrayOpBase { +class ReadFromArrayOp : public ArrayOp { public: ReadFromArrayOp(const std::string &type, const framework::VariableNameMap &inputs, const framework::VariableNameMap &outputs, const framework::AttributeMap &attrs) - : ArrayOpBase(type, inputs, outputs, attrs) {} + : ArrayOp(type, inputs, outputs, attrs) {} void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { auto *x = scope.FindVar(Input("X")); diff --git a/paddle/operators/while_op.cc b/paddle/operators/while_op.cc new file mode 100644 index 0000000000000..4ca6c8507a485 --- /dev/null +++ b/paddle/operators/while_op.cc @@ -0,0 +1,197 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include +#include "paddle/framework/executor.h" +#include "paddle/framework/op_registry.h" +#include "paddle/framework/operator.h" + +namespace paddle { +namespace operators { + +using StepScopeVar = std::vector; +using LoDTensor = framework::LoDTensor; + +constexpr char kStepBlock[] = "step_block"; +constexpr char kCondition[] = "Condition"; +constexpr char kStepScopes[] = "StepScopes"; +constexpr char kParamGrads[] = "X@Grad"; +constexpr char kParameters[] = "X"; + +class WhileOp : public framework::OperatorBase { + public: + WhileOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : framework::OperatorBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + PADDLE_ENFORCE_NOT_NULL(scope.FindVar(Input(kCondition))); + auto &cond = scope.FindVar(Input(kCondition))->Get(); + PADDLE_ENFORCE_EQ(cond.dims(), paddle::framework::make_ddim({1})); + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + auto step_scopes = + scope.FindVar(Output(kStepScopes))->GetMutable(); + + while (cond.data()[0]) { + auto ¤t_scope = scope.NewScope(); + step_scopes->push_back(¤t_scope); + + executor.Run(*program, ¤t_scope, block->ID(), + false /*create_local_scope*/); + } + } +}; + +class WhileOpMaker : public framework::OpProtoAndCheckerMaker { + public: + WhileOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput(kParameters, + "A set of variables, which are required by operators inside the " + "block of While Op.") + .AsDuplicable(); + AddInput( + kCondition, + "(Bool) An scalar. When it's False, the While Op will be terminated.") + .AsDuplicable(); + AddOutput("Out", + "A set of variables, which will be assigned with values " + "generated by perators inside the block of While Op.") + .AsDuplicable(); + AddOutput(kStepScopes, + "(StepScopeVar) A vector of local scope, which size equals the " + "step number of While Op. The i'th scope storages temporary " + "variables generated in the i'th step."); + AddAttr(kStepBlock, + "The step block inside WhileOp"); + AddComment(R"DOC( +)DOC"); + } +}; + +class WhileGradOp : public framework::OperatorBase { + public: + WhileGradOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : framework::OperatorBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + // PADDLE_ENFORCE(...) + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + auto *step_scopes = + scope.FindVar(Input(kStepScopes))->GetMutable(); + + for (auto cur_scope_iter = step_scopes->rbegin(); + cur_scope_iter != step_scopes->rend(); ++cur_scope_iter) { + executor.Run(*program, *cur_scope_iter, block->ID(), false); + + auto &pg_names = Outputs(kParamGrads); + auto &p_names = Inputs(kParameters); + PADDLE_ENFORCE_EQ(pg_names.size(), p_names.size()); + for (size_t prog_id = 0; prog_id < pg_names.size(); ++prog_id) { + auto inside_grad_name = framework::GradVarName(p_names[prog_id]); + + // // TODO(tonyyang-savil: Not sure we need the following + // // If does not compute gradient of that variable inside rnn, + // just + // // continue + // if (local_var_names.find(inside_grad_name) == + // local_var_names.end()) { + // continue; + // } + + // zero gradient variable in step 0 + if (cur_scope_iter == step_scopes->rbegin()) { + auto *var = (*cur_scope_iter)->FindVar(inside_grad_name); + PADDLE_ENFORCE_NOT_NULL(var); + if (var->IsType()) { + auto &inside_tensor = var->Get(); + framework::AttributeMap attrs; + attrs["data_type"] = framework::ToDataType(inside_tensor.type()); + attrs["shape"] = framework::vectorize2int(inside_tensor.dims()); + attrs["value"] = 0.0f; + + auto zero_op = framework::OpRegistry::CreateOp( + "fill_constant", {}, {{"Out", {pg_names[prog_id]}}}, attrs); + zero_op->Run(scope, dev_ctx); + } + } + + // sum gradient + auto *outside_var = scope.FindVar(pg_names[prog_id]); + PADDLE_ENFORCE_NOT_NULL(outside_var); + auto &outside_tensor = *outside_var->GetMutable(); + + std::string result_var_name; + auto *local_result_var = (*cur_scope_iter)->Var(&result_var_name); + auto &local_result_tensor = + *local_result_var->GetMutable(); + + local_result_tensor.ShareDataWith(outside_tensor); + + auto sum_op = framework::OpRegistry::CreateOp( + "sum", {{"X", {result_var_name, inside_grad_name}}}, + {{"Out", {result_var_name}}}, {}); + sum_op->Run(**cur_scope_iter, dev_ctx); + } + } + } +}; + +class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + virtual std::unique_ptr Apply() const { + auto *grad = new framework::OpDescBind(); + grad->SetType("while_grad"); + for (auto &input_param : this->InputNames()) { + grad->SetInput(input_param, this->Input(input_param)); + grad->SetOutput(framework::GradVarName(input_param), + this->InputGrad(input_param)); + } + + for (auto &output_param : this->OutputNames()) { + grad->SetInput(output_param, this->Output(output_param)); + if (output_param != kStepScopes) { + grad->SetInput(framework::GradVarName(output_param), + this->OutputGrad(output_param)); + } + } + grad->SetAttrMap(this->Attrs()); + grad->SetBlockAttr(kStepBlock, *grad_block_[0]); + + return std::unique_ptr(grad); + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(while, paddle::operators::WhileOp, + paddle::operators::WhileOpMaker, + paddle::operators::WhileGradOpDescMaker); diff --git a/paddle/platform/call_once.h b/paddle/platform/call_once.h new file mode 100644 index 0000000000000..d9f49527dcf15 --- /dev/null +++ b/paddle/platform/call_once.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include + +namespace paddle { +namespace platform { + +/* + The current implementation of std::call_once has a bug described in + https://stackoverflow.com/questions/41717579/stdcall-once-hangs-on-second-call-after-callable-threw-on-first-call. + This is likely caused by a deeper bug of pthread_once, which is discussed in + https://patchwork.ozlabs.org/patch/482350/ + + This wrap is a hack to avoid this bug. +*/ +template +inline void call_once(std::once_flag& flag, Callable&& f, Args&&... args) { + bool good = false; + std::exception ex; + std::call_once(flag, + [&](Args&&... args) { + try { + f(args...); + good = true; + } catch (const std::exception& e) { + ex = e; + } catch (...) { + ex = std::runtime_error("excption caught in call_once"); + } + }, + args...); + if (!good) { + throw std::exception(ex); + } +} + +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/dynload/nccl.h b/paddle/platform/dynload/nccl.h index 0618c7414fd12..981b2ab258a34 100644 --- a/paddle/platform/dynload/nccl.h +++ b/paddle/platform/dynload/nccl.h @@ -17,6 +17,7 @@ #include #include #include +#include "paddle/platform/call_once.h" #include "paddle/platform/dynload/dynamic_loader.h" namespace paddle { @@ -27,18 +28,18 @@ extern std::once_flag nccl_dso_flag; extern void* nccl_dso_handle; #ifdef PADDLE_USE_DSO -#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - auto operator()(Args... args) -> decltype(__name(args...)) { \ - using nccl_func = decltype(__name(args...)) (*)(Args...); \ - std::call_once(nccl_dso_flag, \ - paddle::platform::dynload::GetNCCLDsoHandle, \ - &nccl_dso_handle); \ - void* p_##__name = dlsym(nccl_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using nccl_func = decltype(__name(args...)) (*)(Args...); \ + platform::call_once(nccl_dso_flag, \ + paddle::platform::dynload::GetNCCLDsoHandle, \ + &nccl_dso_handle); \ + void* p_##__name = dlsym(nccl_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #else #define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ diff --git a/paddle/platform/transform.h b/paddle/platform/transform.h index f196868c725cb..bb9d59ec0a18c 100644 --- a/paddle/platform/transform.h +++ b/paddle/platform/transform.h @@ -49,8 +49,6 @@ struct Transform { template void operator()(const DeviceContext& context, InputIter first, InputIter last, OutputIter result, UnaryOperation op) { - auto place = context.GetPlace(); - PADDLE_ENFORCE(is_cpu_place(place), "It must use CPU place."); std::transform(first, last, result, op); } @@ -59,8 +57,6 @@ struct Transform { void operator()(const DeviceContext& context, InputIter1 first1, InputIter1 last1, InputIter2 first2, OutputIter result, BinaryOperation op) { - auto place = context.GetPlace(); - PADDLE_ENFORCE(is_cpu_place(place), "It must use CPU place."); std::transform(first1, last1, first2, result, op); } }; diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 0f906e0e470b7..3d8d3f1d2fd39 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -42,6 +42,9 @@ limitations under the License. */ #include "paddle/platform/gpu_info.h" #endif +// disable auto conversion to list in Python +PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray); + namespace paddle { namespace pybind { static size_t UniqueIntegerGenerator(const std::string &prefix) { diff --git a/proto/ModelConfig.proto b/proto/ModelConfig.proto index ebf0911d6ea0b..2c2cc6245932d 100644 --- a/proto/ModelConfig.proto +++ b/proto/ModelConfig.proto @@ -321,6 +321,19 @@ message ClipConfig { required double max = 2; } +message ROIPoolConfig { + required uint32 pooled_width = 1; + required uint32 pooled_height = 2; + required float spatial_scale = 3; + optional uint32 height = 4 [ default = 1 ]; + optional uint32 width = 5 [ default = 1 ]; +} + +message ScaleSubRegionConfig { + required ImageConfig image_conf = 1; + required float value = 2; +} + message LayerInputConfig { required string input_layer_name = 1; optional string input_parameter_name = 2; @@ -342,6 +355,8 @@ message LayerInputConfig { optional MultiBoxLossConfig multibox_loss_conf = 16; optional DetectionOutputConfig detection_output_conf = 17; optional ClipConfig clip_conf = 18; + optional ScaleSubRegionConfig scale_sub_region_conf = 19; + optional ROIPoolConfig roi_pool_conf = 20; } message LayerConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 0e65598485d87..43d02bf70e74c 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1969,6 +1969,18 @@ def __init__(self, name, inputs, size, input_num, num_classes, self.config.size = size +@config_layer('roi_pool') +class ROIPoolLayer(LayerBase): + def __init__(self, name, inputs, pooled_width, pooled_height, spatial_scale, + num_channels, **xargs): + super(ROIPoolLayer, self).__init__(name, 'roi_pool', 0, inputs) + config_assert(len(inputs) == 2, 'ROIPoolLayer must have 2 inputs') + self.config.inputs[0].roi_pool_conf.pooled_width = pooled_width + self.config.inputs[0].roi_pool_conf.pooled_height = pooled_height + self.config.inputs[0].roi_pool_conf.spatial_scale = spatial_scale + self.set_cnn_layer(name, pooled_height, pooled_width, num_channels) + + @config_layer('data') class DataLayer(LayerBase): def __init__(self, @@ -3801,6 +3813,25 @@ def __init__(self, name, inputs, reshape, **xargs): self.config.reshape_conf.width_axis.extend(reshape['width']) +@config_layer('scale_sub_region') +class ScaleSubRegionLayer(LayerBase): + def __init__(self, name, inputs, value, **xargs): + super(ScaleSubRegionLayer, self).__init__( + name, 'scale_sub_region', 0, inputs=inputs, **xargs) + scale_sub_region_conf = self.config.inputs[0].scale_sub_region_conf + scale_sub_region_conf.value = value + + # get channel, width and height from input_0 layer + input_layer = self.get_input_layer(0) + image_conf = scale_sub_region_conf.image_conf + image_conf.img_size = input_layer.width + image_conf.img_size_y = input_layer.height + image_conf.channels = input_layer.size / (input_layer.width * + input_layer.height) + self.set_cnn_layer(name, image_conf.img_size_y, image_conf.img_size, + image_conf.channels) + + # Deprecated, use a new layer specific class instead @config_func def Layer(name, type, **xargs): diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 0fd77a0be6012..617fbff948bf0 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -122,6 +122,7 @@ 'cross_channel_norm_layer', 'multibox_loss_layer', 'detection_output_layer', + 'roi_pool_layer', 'spp_layer', 'pad_layer', 'eos_layer', @@ -144,6 +145,7 @@ 'img_conv3d_layer', 'resize_layer', 'sub_seq_layer', + 'scale_sub_region_layer', ] @@ -220,6 +222,7 @@ class LayerType(object): PRIORBOX_LAYER = 'priorbox' MULTIBOX_LOSS_LAYER = 'multibox_loss' DETECTION_OUTPUT_LAYER = 'detection_output' + ROI_POOL_LAYER = 'roi_pool' CTC_LAYER = 'ctc' WARP_CTC_LAYER = 'warp_ctc' @@ -255,6 +258,8 @@ class LayerType(object): RESIZE = 'resize' SUB_SEQ_LAYER = 'subseq' + SCALE_SUB_REGION_LAYER = 'scale_sub_region' + @staticmethod def is_layer_type(type_name): """ @@ -786,10 +791,9 @@ def __init__(self, name, size, act, bias_attr, layer_attr, parents=None): :type size: int :param act: Activation type. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None @@ -886,10 +890,9 @@ def mixed_layer(size=0, then this function will just return layer's name. :param act: Activation Type. LinearActivation is the default. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: The extra layer config. Default is None. :type layer_attr: ExtraLayerAttribute @@ -1031,10 +1034,9 @@ def fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute|list. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute | None @@ -1305,6 +1307,50 @@ def detection_output_layer(input_loc, name, LayerType.DETECTION_OUTPUT_LAYER, parents=parents, size=size) +@wrap_name_default("roi_pool") +def roi_pool_layer(input, + rois, + pooled_width, + pooled_height, + spatial_scale, + num_channels=None, + name=None): + """ + A layer used by Fast R-CNN to extract feature maps of ROIs from the last + feature map. + + :param name: The Layer Name. + :type name: basestring + :param input: The input layer. + :type input: LayerOutput. + :param rois: The input ROIs' data. + :type rois: LayerOutput. + :param pooled_width: The width after pooling. + :type pooled_width: int + :param pooled_height: The height after pooling. + :type pooled_height: int + :param spatial_scale: The spatial scale between the image and feature map. + :type spatial_scale: float + :param num_channels: number of input channel. + :type num_channels: int + :return: LayerOutput + """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + size = num_channels * pooled_width * pooled_height + Layer( + name=name, + type=LayerType.ROI_POOL_LAYER, + inputs=[input.name, rois.name], + pooled_width=pooled_width, + pooled_height=pooled_height, + spatial_scale=spatial_scale, + num_channels=num_channels) + return LayerOutput( + name, LayerType.ROI_POOL_LAYER, parents=[input, rois], size=size) + + @wrap_name_default("cross_channel_norm") def cross_channel_norm_layer(input, name=None, param_attr=None): """ @@ -1387,10 +1433,9 @@ def pooling_layer(input, :type pooling_type: BasePoolingType | None :param stride: The step size between successive pooling regions. :type stride: Int - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: The Extra Attributes for layer, such as dropout. :type layer_attr: ExtraLayerAttribute | None @@ -1488,10 +1533,9 @@ def lstmemory(input, :type gate_act: BaseActivation :param state_act: state activation type, TanhActivation by default. :type state_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute | None | False @@ -1614,10 +1658,9 @@ def grumemory(input, This activation affects the :math:`z_t` and :math:`r_t`. It is the :math:`\\sigma` in the above formula. :type gate_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute | None | False @@ -1814,10 +1857,9 @@ def expand_layer(input, :type expand_as: LayerOutput :param name: The name of this layer. It is optional. :type name: basestring - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param expand_level: whether input layer is timestep(default) or sequence. :type expand_level: ExpandLevel @@ -1936,10 +1978,9 @@ def seq_reshape_layer(input, :type act: BaseActivation :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :return: LayerOutput object. :rtype: LayerOutput @@ -2323,10 +2364,9 @@ def hsigmoid(input, :type num_classes: int | None :param name: The name of this layer. It is optional. :type name: basestring - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: Parameter Attribute. None means default parameter. :type param_attr: ParameterAttribute | None @@ -2466,10 +2506,9 @@ def img_conv_layer(input, :type dilation: int | tuple | list :param dilation_y: The y dimension of the dilation. :type dilation_y: int - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param num_channels: number of input channels. If None will be set automatically from previous output. @@ -3216,10 +3255,9 @@ def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): :type input: LayerOutput | list | tuple :param act: Activation Type. LinearActivation is the default. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute @@ -3372,10 +3410,9 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None, :type act: BaseActivation :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :return: LayerOutput object. :rtype: LayerOutput @@ -3555,10 +3592,9 @@ def lstm_step_layer(input, :type gate_act: BaseActivation :param state_act: State Activation Type. TanhActivation is the default. :type state_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: layer's extra attribute. :type layer_attr: ExtraLayerAttribute @@ -3614,10 +3650,9 @@ def gru_step_layer(input, :param name: The name of this layer. It is optional. :param gate_act: Activation type of this layer's two gates. Default is Sigmoid. :type gate_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: the parameter_attribute for transforming the output_mem from previous step. @@ -3677,10 +3712,9 @@ def gru_step_naive_layer(input, :type act: BaseActivation :param gate_act: Activation type of this layer's two gates. Default is Sigmoid. :type gate_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: :param layer_attr: @@ -3810,10 +3844,9 @@ def recurrent_layer(input, :type input: LayerOutput :param act: Activation type. TanhActivation is the default. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: parameter attribute. :type param_attr: ParameterAttribute @@ -4803,10 +4836,9 @@ def tensor_layer(a, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute | None @@ -4868,10 +4900,9 @@ def selective_fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute | None @@ -5494,7 +5525,11 @@ def crf_decoding_layer(input, return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1) -@wrap_act_default(act=SigmoidActivation()) +""" +Following are cost Layers. +""" + + @wrap_bias_attr_default(has_bias=True) @wrap_param_attr_default() @wrap_name_default() @@ -5502,7 +5537,6 @@ def crf_decoding_layer(input, def nce_layer(input, label, num_classes=None, - act=None, param_attr=None, weight=None, num_neg_samples=10, @@ -5511,9 +5545,12 @@ def nce_layer(input, bias_attr=None, layer_attr=None): """ - Noise-contrastive estimation. - Implements the method in the following paper: - A fast and simple algorithm for training neural probabilistic language models. + Noise-contrastive estimation. This layer implements the method in the + following paper: + + Reference: + A fast and simple algorithm for training neural probabilistic language + models. https://www.cs.toronto.edu/~amnih/papers/ncelm.pdf The example usage is: @@ -5525,32 +5562,37 @@ def nce_layer(input, :param name: The name of this layer. It is optional. :type name: basestring - :param input: The input layers. It could be a LayerOutput of list/tuple of LayerOutput. + :param input: The input layers. It should be a LayerOutput or a list/tuple + of LayerOutput. :type input: LayerOutput | list | tuple | collections.Sequence - :param label: label layer + :param label: The ground truth. :type label: LayerOutput - :param weight: weight layer, can be None(default) + :param weight: The weight layer defines a weight for each sample in the + mini-batch. The default value is None. :type weight: LayerOutput - :param num_classes: number of classes. + :param num_classes: The class number. :type num_classes: int - :param act: Activation type. SigmoidActivation is the default. - :type act: BaseActivation - :param param_attr: The Parameter Attribute|list. - :type param_attr: ParameterAttribute - :param num_neg_samples: number of negative samples. Default is 10. + :param param_attr: The parameter attributes. + :type param_attr: ParameterAttribute|list + :param num_neg_samples: The number of sampled negative labels. The default + value is 10. :type num_neg_samples: int - :param neg_distribution: The distribution for generating the random negative labels. - A uniform distribution will be used if not provided. - If not None, its length must be equal to num_classes. + :param neg_distribution: The discrete noisy distribution over the output + space from which num_neg_samples negative labels + are sampled. If this parameter is not set, a + uniform distribution will be used. A user defined + distribution is a list whose length must be equal + to the num_classes. Each member of the list defines + the probability of a class given input x. :type neg_distribution: list | tuple | collections.Sequence | None - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The attribute for bias. If this parameter is set False or + any object whose type is not ParameterAttribute, no bias + is added. If this parameter is set True, the bias is + initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :return: layer name. + :return: The LayerOutput object. :rtype: LayerOutput """ if isinstance(input, LayerOutput): @@ -5573,8 +5615,6 @@ def nce_layer(input, assert isinstance(neg_distribution, collections.Sequence) assert len(neg_distribution) == num_classes assert abs(sum(neg_distribution) - 1.0) < 1e-5 - if not isinstance(act, BaseActivation): - raise TypeError() ipts_for_layer = [] parents = [] @@ -5596,7 +5636,7 @@ def nce_layer(input, type=LayerType.NCE_LAYER, num_classes=num_classes, neg_sampling_dist=neg_distribution, - active_type=act.name, + active_type=SigmoidActivation().name, num_neg_samples=num_neg_samples, inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), @@ -5606,12 +5646,7 @@ def nce_layer(input, LayerType.NCE_LAYER, parents=parents, size=l.config.size, - activation=act) - - -""" -following are cost Layers. -""" + activation=SigmoidActivation()) @wrap_name_default() @@ -5770,20 +5805,21 @@ def cross_entropy(input, :param input: The first input layer. :type input: LayerOutput. :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param coeff: The cost is multiplied with coeff. - The coefficient affects the gradient in the backward. - :type coeff: float. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float :param weight: The cost of each sample is multiplied with each weight. The weight should be a layer with size=1. Note that gradient will not be calculated for weight. :type weight: LayerOutout - :param layer_attr: Extra Layer Attribute. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. - :rtype: LayerOutput. + :rtype: LayerOutput """ ipts, parents = __cost_input__(input, label, weight) @@ -5816,19 +5852,21 @@ def cross_entropy_with_selfnorm(input, label=label_layer) :param input: The first input layer. - :type input: LayerOutput. + :type input: LayerOutput :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float :param softmax_selfnorm_alpha: The scale factor affects the cost. - :type softmax_selfnorm_alpha: float. - :param layer_attr: Extra Layer Attribute. + :type softmax_selfnorm_alpha: float + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. - :rtype: LayerOutput. + :rtype: LayerOutput """ Layer( name=name, @@ -5849,7 +5887,7 @@ def cross_entropy_with_selfnorm(input, @layer_support() def sum_cost(input, name=None, layer_attr=None): """ - A loss layer which calculate the sum of the input as loss + A loss layer which calculates the sum of the input as loss. The example usage is: @@ -5858,10 +5896,11 @@ def sum_cost(input, name=None, layer_attr=None): cost = sum_cost(input=input_layer) :param input: The input of this layer. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param layer_attr: Extra Layer Attribute. + :type name: basestring + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput. @@ -5901,16 +5940,18 @@ def huber_regression_cost(input, cost = huber_regression_cost(input=input_layer, label=label_layer) :param input: The first input layer. - :type input: LayerOutput. + :type input: LayerOutput :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. + :type name: basestring :param delta: The difference between the observed and predicted values. - :type delta: float. - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float. - :param layer_attr: Extra Layer Attribute. + :type delta: float + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput. @@ -5951,17 +5992,19 @@ def huber_classification_cost(input, cost = huber_classification_cost(input=input_layer, label=label_layer) :param input: The first input layer. - :type input: LayerOutput. + :type input: LayerOutput :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float. - :param layer_attr: Extra Layer Attribute. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. - :rtype: LayerOutput. + :rtype: LayerOutput """ assert isinstance(input, LayerOutput) if input.size is not None: @@ -5998,10 +6041,12 @@ def multi_binary_label_cross_entropy(input, :param label: The input label. :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring - :param coeff: The coefficient affects the gradient in the backward. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. :type coeff: float - :param layer_attr: Extra Layer Attribute. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput @@ -6104,7 +6149,7 @@ def cross_entropy_over_beam(input, name=None): :param input: Input beams for this layer. :type input: BeamInput - :param name: The name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6139,7 +6184,7 @@ def cross_entropy_over_beam(input, name=None): def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): """ This is a L1 loss but more smooth. It requires that the - size of input and label are equal. The formula is as follows, + sizes of input and label are equal. The formula is as follows, .. math:: @@ -6151,8 +6196,9 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): smooth_{L1}(x) = \\begin{cases} 0.5x^2& \\text{if} \\ |x| < 1 \\\\ |x|-0.5& \\text{otherwise} \end{cases} - More details can be found by referring to `Fast R-CNN - `_ + Reference: + Fast R-CNN + https://arxiv.org/pdf/1504.08083v2.pdf The example usage is: @@ -6166,10 +6212,12 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): :param label: The input label. :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring - :param coeff: The coefficient affects the gradient in the backward. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. :type coeff: float - :param layer_attr: Extra Layer Attribute. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput @@ -6191,12 +6239,12 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): @wrap_name_default() def multiplex_layer(input, name=None, layer_attr=None): """ - This layer multiplex multiple layers according to the index, - which is provided by the first input layer. - inputs[0]: the index of the layer to output of size batchSize. + This layer multiplex multiple layers according to the indexes, + which are provided by the first input layer. + inputs[0]: the indexes of the layers to form the output of size batchSize. inputs[1:N]; the candidate output data. - For each index i from 0 to batchSize -1, the output is the i-th row of the - (index[i] + 1)-th layer. + For each index i from 0 to batchSize - 1, the i-th row of the output is the + the same to the i-th row of the (index[i] + 1)-th layer. For each i-th row of output: .. math:: @@ -6215,7 +6263,8 @@ def multiplex_layer(input, name=None, layer_attr=None): :type input: list of LayerOutput :param name: The name of this layer. It is optional. :type name: basestring - :param layer_attr: extra layer attributes. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute. :return: LayerOutput object. :rtype: LayerOutput @@ -6319,14 +6368,14 @@ def row_conv_layer(input, :type context_len: int :param act: Activation Type. LinearActivation is the default. :type act: BaseActivation - :param param_attr: The Parameter Attribute. If None, the parameter will be - initialized smartly. It's better to set it by yourself. + :param param_attr: The parameter attribute. See ParameterAttribute for + details. :type param_attr: ParameterAttribute - :param layer_attr: Extra Layer config. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput - """ assert isinstance(input, LayerOutput) assert context_len > 0, "the context_len must be greatet than 0." @@ -6351,7 +6400,7 @@ def prelu_layer(input, param_attr=None, layer_attr=None): """ - The Parameter Relu activation that actives outputs with a learnable weight. + The Parametric Relu activation that actives outputs with a learnable weight. Reference: Delving Deep into Rectifiers: Surpassing Human-Level Performance on @@ -6371,16 +6420,17 @@ def prelu_layer(input, :type name: basestring :param input: The input of this layer. :type input: LayerOutput - :param partial_sum: this parameter makes a group of inputs share a same weight. + :param partial_sum: this parameter makes a group of inputs share the same weight. - partial_sum = 1, indicates the element-wise activation: each element has a weight. - - partial_sum = number of elements in one channel, indicates the channel-wise activation, elements in a channel share a same weight. - - partial_sum = number of outputs, indicates all elements share a same weight. + - partial_sum = number of elements in one channel, indicates the channel-wise activation, elements in a channel share the same weight. + - partial_sum = number of outputs, indicates all elements share the same weight. :type partial_sum: int :param param_attr: The parameter attribute. See ParameterAttribute for details. - :type param_attr: ParameterAttribute | None - :param layer_attr: Extra layer configurations. Default is None. + :type param_attr: ParameterAttribute + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput @@ -6436,34 +6486,34 @@ def gated_unit_layer(input, :param input: The input of this layer. :type input: LayerOutput - :param size: output size of the gated unit. + :param size: The dimension of this layer's output. :type size: int - :param act: Activation type of the projected input. LinearActivation is the default. + :param act: Activation type of the projection. LinearActivation is the default. :type act: BaseActivation :param name: The name of this layer. It is optional. :type name: basestring - :param gate_attr: Attributes to tune the gate output, for example, error - clipping threshold, dropout and so on. See ExtraLayerAttribute for - more details. + :param gate_attr: The extra layer attribute of the gate. See ExtraLayerAttribute for + details. :type gate_attr: ExtraLayerAttribute | None - :param gate_param_attr: Attributes to tune the learnable projected matrix - parameter of the gate. - :type gate_param_attr: ParameterAttribute | None - :param gate_bias_attr: Attributes to tune the learnable bias of the gate. - :type gate_bias_attr: ParameterAttribute | None - :param inproj_attr: Attributes to the tune the projected input, for - example, error clipping threshold, dropout and so on. See - ExtraLayerAttribute for more details. + :param gate_param_attr: The parameter attribute of the gate. See ParameterAttribute + for details. + :type gate_param_attr: ParameterAttribute + :param gate_bias_attr: The bias attribute of the gate. If the parameter is set to False or + an object whose type is not ParameterAttribute, no bias is defined. + If the parameter is set to True, the bias is initialized to zero. + :type gate_bias_attr: ParameterAttribute | bool | None | Any + :param inproj_attr: Extra layer attributes of the projection. See ExtraLayerAttribute for + details. :type inproj_attr: ExtraLayerAttribute | None - :param inproj_param_attr: Attributes to tune the learnable parameter of - the projection of input. - :type inproj_param_attr: ParameterAttribute | None - :param inproj_bias_attr: Attributes to tune the learnable bias of - projection of the input. - :type inproj_bias_attr: ParameterAttribute | None - :param layer_attr: Attributes to tune the final output of the gated unit, - for example, error clipping threshold, dropout and so on. See - ExtraLayerAttribute for more details. + :param inproj_param_attr: The parameter attribute of the projection. See ParameterAttribute + for details. + :type inproj_param_attr: ParameterAttribute + :param inproj_bias_attr: The bias attribute of the projection. If the parameter is set to False + or an object whose type is not ParameterAttribute, no bias is defined. + If the parameter is set to True, the bias is initialized to zero. + :type inproj_bias_attr: ParameterAttribute | bool | None | Any + :param layer_attr: Extra layer attribute of the product. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput @@ -6659,9 +6709,9 @@ def clip_layer(input, min, max, name=None): :param input: The input of this layer. :type input: LayerOutput. :param min: The lower threshold for clipping. - :type min: double + :type min: float :param max: The upper threshold for clipping. - :type max: double + :type max: float :return: LayerOutput object. :rtype: LayerOutput """ @@ -6709,7 +6759,6 @@ def seq_slice_layer(input, starts, ends, name=None): :type ends: LayerOutput | None :return: LayerOutput object. :rtype: LayerOutput - """ assert isinstance(input, LayerOutput), ( @@ -6830,20 +6879,21 @@ def img_conv3d_layer(input, :param padding: The numbers of padding along three axises. If the parameter is set to one integer, they will be same. :type padding: int | tuple | list - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param num_channels: The number of input channels. If the parameter is not set or set to None, its actual value will be automatically set to the channels number of the input . :type num_channels: int - :param param_attr: The parameter attribute of the convolution. + :param param_attr: The parameter attribute of the convolution. See ParameterAttribute for + details. :type param_attr: ParameterAttribute :param shared_biases: Whether biases will be shared between filters or not. :type shared_biases: bool - :param layer_attr: Extra layer attributes. + :param layer_attr: The extra layer attributes. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :param trans: True if it is a convTransLayer, False if it is a convLayer :type trans: bool @@ -6950,12 +7000,12 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): :type name: basestring :param input: The input of this layer. :type input: LayerOutput - :param param_attr: The parameter attribute of scaling. + :param param_attr: The parameter attribute of scaling. See ParameterAttribute for + details. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :return: LayerOutput object. :rtype: LayerOutput @@ -7013,10 +7063,9 @@ def sub_seq_layer(input, offsets, sizes, act=None, bias_attr=None, name=None): :type sizes: LayerOutput :param act: Activation type, LinearActivation is the default. :type act: BaseActivation. - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :return: LayerOutput object. :rtype: LayerOutput @@ -7042,3 +7091,54 @@ def sub_seq_layer(input, offsets, sizes, act=None, bias_attr=None, name=None): LayerType.SUB_SEQ_LAYER, parents=[input, offsets, sizes], size=input.size) + + +@wrap_name_default('scale_sub_region') +def scale_sub_region_layer(input, indices, value, name=None): + """ + Given an image or feature map with CHW information, scale_sub_region_layer + can be used to multiply a real value to values of a sub continuous region. + You can provide start and end indices of CHW for each instance. + Please notice that all start indices are counting from 1. + The shape of indices should be [batch_size, 6] and the layout for each row + is [C_Start, C_End, H_Start, H_End, W_Start, W_End]. + + .. code-block:: python + + scale_sub_region = scale_sub_region_layer(input=input, + indices=indices, + value=value) + + :param name: The name of this layer. It is optional. + :type name: basestring + :param input: The input of this layer which should contains CHW information. + :type input: LayerOutput + :param indices: Start index and end index for C H W, the input value should + be a 2-D matrix with shape [batch_size, 6]. + :type indices: LayerOutput. + :param value: value to multiply. + :type value: float + :return: LayerOutput object. + :rtype: LayerOutput + """ + + assert isinstance(input, LayerOutput), ( + 'The first input of scale_sub_region_layer, ' + 'must be a PaddlePaddle layer.') + assert isinstance(indices, LayerOutput), ( + 'The start and end indices for CHW, must be a PaddlePaddle layer.') + assert isinstance(value, float), ( + 'The value to multiply, must be a real value.') + + Layer( + name=name, + type=LayerType.SCALE_SUB_REGION_LAYER, + inputs=[input.name, indices.name], + value=value) + + return LayerOutput( + name, + LayerType.SCALE_SUB_REGION_LAYER, + parents=[input, indices], + num_filters=input.num_filters, + size=input.size) diff --git a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh index 6a4550c209762..1c7451e0abf5d 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh @@ -9,7 +9,7 @@ test_seq_concat_reshape test_pad test_smooth_l1 test_multiplex_layer test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_layer test_recursive_topology test_gated_unit_layer test_clip_layer test_row_l2_norm_layer test_kmax_seq_socre_layer test_sub_nested_seq_select_layer test_scale_shift_layer -test_seq_slice_layer test_cross_entropy_over_beam test_pooling3D_layer -test_conv3d_layer test_deconv3d_layer test_BatchNorm3D test_resize_layer) +test_seq_slice_layer test_cross_entropy_over_beam test_roi_pool_layer test_pooling3D_layer +test_conv3d_layer test_deconv3d_layer test_BatchNorm3D test_resize_layer test_scale_sub_region_layer) export whole_configs=(test_split_datasource) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_roi_pool_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_roi_pool_layer.protostr new file mode 100644 index 0000000000000..f1bc65b3aee74 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_roi_pool_layer.protostr @@ -0,0 +1,98 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 588 + active_type: "" + height: 14 + width: 14 +} +layers { + name: "rois" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconv" + size: 3136 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 3 + channels: 3 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 3 + output_x: 14 + img_size: 14 + caffe_mode: true + filter_size_y: 3 + padding_y: 1 + stride_y: 1 + output_y: 14 + img_size_y: 14 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 16 + shared_biases: true + height: 14 + width: 14 +} +layers { + name: "__roi_pool_0__" + type: "roi_pool" + size: 784 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + roi_pool_conf { + pooled_width: 7 + pooled_height: 7 + spatial_scale: 0.0625 + } + } + inputs { + input_layer_name: "rois" + } + height: 7 + width: 7 +} +parameters { + name: "___conv_0__.w0" + size: 432 + initial_mean: 0.0 + initial_std: 0.272165526976 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 16 + initial_mean: 0.0 + initial_std: 0.0 + dims: 16 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +input_layer_names: "rois" +output_layer_names: "__roi_pool_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "rois" + layer_names: "__conv_0__" + layer_names: "__roi_pool_0__" + input_layer_names: "data" + input_layer_names: "rois" + output_layer_names: "__roi_pool_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_sub_region_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_sub_region_layer.protostr new file mode 100644 index 0000000000000..d20133a10ec60 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_sub_region_layer.protostr @@ -0,0 +1,51 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 2016 + active_type: "" + height: 48 + width: 42 +} +layers { + name: "indices" + type: "data" + size: 6 + active_type: "" +} +layers { + name: "__scale_sub_region_0__" + type: "scale_sub_region" + size: 2016 + active_type: "" + inputs { + input_layer_name: "data" + scale_sub_region_conf { + image_conf { + channels: 1 + img_size: 42 + img_size_y: 48 + } + value: 0.0 + } + } + inputs { + input_layer_name: "indices" + } + height: 48 + width: 42 +} +input_layer_names: "data" +input_layer_names: "indices" +output_layer_names: "__scale_sub_region_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "indices" + layer_names: "__scale_sub_region_0__" + input_layer_names: "data" + input_layer_names: "indices" + output_layer_names: "__scale_sub_region_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py new file mode 100644 index 0000000000000..b739a81b8505c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py @@ -0,0 +1,23 @@ +from paddle.trainer_config_helpers import * + +data = data_layer(name='data', size=3 * 14 * 14, height=14, width=14) + +rois = data_layer(name='rois', size=10) + +conv = img_conv_layer( + input=data, + filter_size=3, + num_channels=3, + num_filters=16, + padding=1, + act=LinearActivation(), + bias_attr=True) + +roi_pool = roi_pool_layer( + input=conv, + rois=rois, + pooled_width=7, + pooled_height=7, + spatial_scale=1. / 16) + +outputs(roi_pool) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py new file mode 100644 index 0000000000000..8d4bf28bf1eaf --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=1000, learning_rate=1e-5) + +data = data_layer(name='data', size=2016, height=48, width=42) +indices = data_layer(name='indices', size=6) + +scale_sub_region = scale_sub_region_layer( + input=data, indices=indices, value=0.0) + +outputs(scale_sub_region) diff --git a/python/paddle/v2/dataset/uci_housing.py b/python/paddle/v2/dataset/uci_housing.py index ce60aa21c2ad1..98b97c75ca72f 100644 --- a/python/paddle/v2/dataset/uci_housing.py +++ b/python/paddle/v2/dataset/uci_housing.py @@ -22,6 +22,7 @@ import numpy as np import os import paddle.v2.dataset.common +from paddle.v2.parameters import Parameters __all__ = ['train', 'test'] @@ -34,7 +35,8 @@ UCI_TRAIN_DATA = None UCI_TEST_DATA = None - +URL_MODEL = 'https://github.com/PaddlePaddle/book/raw/develop/01.fit_a_line/fit_a_line.tar' +MD5_MODEL = '52fc3da8ef3937822fcdd87ee05c0c9b' def feature_range(maximums, minimums): import matplotlib @@ -111,6 +113,13 @@ def reader(): return reader +def model(): + tar_file = paddle.v2.dataset.common.download(URL_MODEL, 'fit_a_line.tar', MD5_MODEL) + with open(tar_file, 'r') as f: + parameters = Parameters.from_tar(f) + return parameters + + def fetch(): paddle.v2.dataset.common.download(URL, 'uci_housing', MD5) diff --git a/python/paddle/v2/framework/framework.py b/python/paddle/v2/framework/framework.py index 8fb3cca91e5f8..0e6f083e5ba25 100644 --- a/python/paddle/v2/framework/framework.py +++ b/python/paddle/v2/framework/framework.py @@ -285,7 +285,7 @@ def find_name(var_list, name): self.desc.check_attrs() no_kernel_op_set = { 'feed', 'fetch', 'save', 'load', 'recurrent', - 'rnn_memory_helper_grad' + 'rnn_memory_helper_grad', 'conditional_block', 'while' } if type not in no_kernel_op_set: self.desc.infer_var_type(self.block.desc) diff --git a/python/paddle/v2/framework/layer_helper.py b/python/paddle/v2/framework/layer_helper.py index c38346b79fecf..552976185dfc2 100644 --- a/python/paddle/v2/framework/layer_helper.py +++ b/python/paddle/v2/framework/layer_helper.py @@ -4,7 +4,7 @@ from paddle.v2.framework.framework import Variable, g_main_program, \ g_startup_program, unique_name, Program from paddle.v2.framework.initializer import ConstantInitializer, \ - UniformInitializer + UniformInitializer, XavierInitializer class LayerHelper(object): @@ -61,7 +61,7 @@ def input(self, input_param_name='input'): @property def param_attr(self): - default = {'name': None, 'initializer': UniformInitializer()} + default = {'name': None, 'initializer': XavierInitializer()} actual = self.kwargs.get('param_attr', None) if actual is None: actual = default @@ -70,10 +70,11 @@ def param_attr(self): actual[default_field] = default[default_field] return actual + @property def bias_attr(self): - default = {'name': None, 'initializer': ConstantInitializer()} + default = {'name': None, 'initializer': XavierInitializer()} bias_attr = self.kwargs.get('bias_attr', None) - if bias_attr is True: + if bias_attr is None: bias_attr = default if isinstance(bias_attr, dict): @@ -166,7 +167,7 @@ def append_bias_op(self, input_var, num_flatten_dims=None): num_flatten_dims = 1 size = list(input_var.shape[num_flatten_dims:]) - bias_attr = self.bias_attr() + bias_attr = self.bias_attr if not bias_attr: return input_var diff --git a/python/paddle/v2/framework/layers.py b/python/paddle/v2/framework/layers.py index cb9955f6e370d..6ec6eab59e6bd 100644 --- a/python/paddle/v2/framework/layers.py +++ b/python/paddle/v2/framework/layers.py @@ -1,33 +1,59 @@ import paddle.v2.framework.core as core +import paddle.v2.framework.proto.framework_pb2 as framework_pb2 from paddle.v2.framework.framework import OpProtoHolder, Variable, Program, \ Operator from paddle.v2.framework.initializer import ConstantInitializer, \ NormalInitializer from paddle.v2.framework.layer_helper import LayerHelper, unique_name import re +import cStringIO __all__ = [ 'fc', 'data', 'cross_entropy', 'conv2d', 'pool2d', 'embedding', 'concat', 'StaticRNN', 'cast', 'sequence_conv', 'sequence_pool', 'sums', 'cos_sim', - 'batch_norm', 'accuracy' + 'batch_norm', 'accuracy', 'split_lod_tensor' ] def fc(input, size, param_attr=None, - bias_attr=True, + bias_attr=None, name=None, act=None, num_flatten_dims=1, main_program=None, startup_program=None): - # create helper + """ + Fully Connected Layer. + + Args: + input: The input tensor to the function + size: The size of the layer + param_attr: The parameters/weights to the FC Layer + bias_attr: The bias parameter for the FC layer + name: Name/alias of the function + act: Activation to be applied to the output of FC layer + num_flatten_dims: Number of columns in input + main_program: Name of the main program that calls this + startup_program: Name of the startup program + + This function can take in multiple inputs and performs the Fully Connected + function (linear transformation) on top of each of them. + So for input x, the output will be : Wx + b. Where W is the parameter, + b the bias and x is the input. + + The function also applies an activation (non-linearity) on top of the + output, if activation is passed in the input. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ helper = LayerHelper('fc', **locals()) dtype = helper.input_dtype() - # mul mul_results = [] for input_var, param_attr in helper.iter_inputs_and_params(): input_shape = input_var.shape @@ -68,6 +94,26 @@ def embedding(input, param_attr=None, main_program=None, startup_program=None): + """ + Embedding Layer. + + Args: + input: The input to the function + size: The size of the layer + data_type: The type of data : float32, float_16, int etc + is_sparse: A flag that decleares whether the input is sparse + param_attr: Parameters for this layer + main_program: Name of the main program that calls this + startup_program: Name of the startup program + + This function can take in the input (which is a vector of IDs) and + performs a lookup in the lookup_table using these IDs, to result into + the embedding of each ID in the input. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ helper = LayerHelper('embedding', **locals()) w = helper.create_parameter( attr=helper.param_attr, shape=size, dtype=data_type) @@ -81,13 +127,85 @@ def embedding(input, return tmp +# TODO(qijun): expose H0 and C0 +def dynamic_lstm(input, + size, + data_type='float32', + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + main_program=None, + startup_program=None): + helper = LayerHelper('lstm', **locals()) + size = size / 4 + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 4 * size], dtype=data_type) + bias_size = [1, 7 * size] + if not use_peepholes: + bias_size[1] = 4 * size + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=data_type, suffix='b') + + hidden = helper.create_tmp_variable(data_type) + cell = helper.create_tmp_variable(data_type) + batch_gate = helper.create_tmp_variable(data_type) + batch_cell_pre_act = helper.create_tmp_variable(data_type) + + helper.append_op( + type='lstm', + inputs={'Input': input, + 'Weight': weight, + 'Bias': bias}, + outputs={ + 'Hidden': hidden, + 'Cell': cell, + 'BatchGate': batch_gate, + 'BatchCellPreAct': batch_cell_pre_act + }, + attrs={ + 'use_peepholes': use_peepholes, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation + }) + return hidden, cell + + def data(name, shape, data_type='float32', type=core.VarDesc.VarType.LOD_TENSOR, append_batch_size=True, main_program=None, - startup_program=None): + startup_program=None, + stop_gradient=True): + """ + Data Layer. + + Args: + name: The name/alias of the function + shape: Tuple declaring the shape. + data_type: The type of data : float32, float_16, int etc + type: The output type. By default it is LOD_TENSOR. + append_batch_size: Whether or not to append the data as a batch. + main_program: Name of the main program that calls this + startup_program: Name of the startup program + stop_gradient: A boolean that mentions whether gradient should flow. + + This function takes in input and based on whether data has + to be returned back as a minibatch, it creates the global variable using + the helper functions. The global variables can be accessed by all the + following operations and layers in the graph. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ helper = LayerHelper('data', **locals()) shape = list(shape) for i in xrange(len(shape)): @@ -101,15 +219,97 @@ def data(name, shape = [-1] + shape # append batch size as -1 return helper.create_global_variable( - name=name, shape=shape, dtype=data_type, type=type, stop_gradient=True) + name=name, + shape=shape, + dtype=data_type, + type=type, + stop_gradient=stop_gradient) + + +def create_tensor(dtype, name=None, main_program=None): + helper = LayerHelper("create_tensor", **locals()) + return helper.create_variable(name=helper.name, dtype=dtype) def _convert_(name): + """ + Formatting. + + Args: + name: The name/alias + + This function takes in a name and converts it to a standard format of + group1_group2. Where as per the regular expression, group1 can have + alphabets and numbers and group2 has capital alphabets. + + """ s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() +def _generate_doc_string_(op_proto): + """ + Generate docstring by OpProto + + Args: + op_proto (framework_pb2.OpProto): a protobuf message typed OpProto + + Returns: + str: the document string + """ + + def _type_to_str_(tp): + return framework_pb2.AttrType.Name(tp) + + if not isinstance(op_proto, framework_pb2.OpProto): + raise TypeError("OpProto should be `framework_pb2.OpProto`") + + buf = cStringIO.StringIO() + buf.write(op_proto.comment) + buf.write('\nArgs:\n') + for each_input in op_proto.inputs: + line_begin = ' {0}: '.format(_convert_(each_input.name)) + buf.write(line_begin) + buf.write(each_input.comment) + buf.write('\n') + buf.write(' ' * len(line_begin)) + buf.write('Duplicable: ') + buf.write(str(each_input.duplicable)) + buf.write(' Optional: ') + buf.write(str(each_input.dispensable)) + buf.write('\n') + + for each_attr in op_proto.attrs: + buf.write(' ') + buf.write(each_attr.name) + buf.write(' (') + buf.write(_type_to_str_(each_attr.type)) + buf.write('): ') + buf.write(each_attr.comment) + buf.write('\n') + + if len(op_proto.outputs) != 0: + buf.write('\nReturns:\n') + buf.write(' ') + for each_opt in op_proto.outputs: + if not each_opt.intermediate: + break + buf.write(each_opt.comment) + + return buf.getvalue() + + def _create_op_func_(op_type): + """ + Create an Operator for a Function. + + Args: + op_type: The name of the operator to be created + + This function takes in the operator type (sigmoid, mean , average etc) and + creates the operator functionality. + + """ op_proto = OpProtoHolder.instance().get_op_proto(op_type) not_intermediate_outputs = \ filter(lambda output: not output.intermediate, op_proto.outputs) @@ -117,26 +317,26 @@ def _create_op_func_(op_type): filter(lambda output: output.intermediate, op_proto.outputs) if len(not_intermediate_outputs) != 1: - raise ValueError( - "Only one not intermediate output operator can be automatically generated" - ) + raise ValueError("Only one non intermediate output operator can be", + "automatically generated") if not_intermediate_outputs[0].duplicable: raise ValueError( - "Only not duplicable op can be automatically generated") + "Only non duplicable op can be automatically generated") for output in intermediate_outputs: if output.duplicable: - raise ValueError( - "Only when all intermediate ops are not duplicable, " - "this op can be automatically generated") + raise ValueError("The op can be automatically generated only when ", + "all intermediate ops are not duplicable") o_name = not_intermediate_outputs[0].name intermediate_output_names = [output.name for output in intermediate_outputs] - def func(**kwargs): - helper = LayerHelper(op_type, **kwargs) - inputs = dict() + def infer_and_check_data_type(op_proto, **kwargs): + """ + This function performs the sanity check for data_type and + instance type. + """ dtype = None for ipt in op_proto.inputs: name = _convert_(ipt.name) @@ -153,6 +353,20 @@ def func(**kwargs): elif dtype != each.data_type: raise ValueError( "operator {0} must input same dtype".format(op_type)) + + return dtype + + def func(**kwargs): + helper = LayerHelper(op_type, **kwargs) + + dtype = infer_and_check_data_type(op_proto, **kwargs) + + inputs = dict() + for ipt in op_proto.inputs: + name = _convert_(ipt.name) + val = kwargs.pop(name, []) + if not isinstance(val, list) and not isinstance(val, tuple): + val = [val] inputs[ipt.name] = val outputs = dict() @@ -166,6 +380,7 @@ def func(**kwargs): func.__name__ = op_type globals()[op_type] = func + func.__doc__ = _generate_doc_string_(op_proto) global __all__ __all__.append(op_type) @@ -178,9 +393,32 @@ def func(**kwargs): _create_op_func_('elementwise_add') _create_op_func_('sigmoid') _create_op_func_('scale') +_create_op_func_('reshape') +_create_op_func_('transpose') + + +def fill_constant(data_type, shape, value=None, program=None): + """ + This function creates a tensor , with shape as mentioned in the input and + specified data_type and fills this up with a constant value that + comes in the input. + """ + helper = LayerHelper('fill_constant', **locals()) + out = helper.create_tmp_variable(dtype=data_type) + helper.append_op( + type='fill_constant', + outputs={'Out': [out]}, + attrs={'data_type': data_type, + 'shape': shape, + 'value': value}) + return out def cast(x, data_type, main_program=None): + """ + This function takes in the input with input_data_type + and casts it to the output_data_type as the output. + """ helper = LayerHelper('cast', **locals()) out = helper.create_tmp_variable(dtype=data_type) helper.append_op( @@ -193,6 +431,10 @@ def cast(x, data_type, main_program=None): def concat(input, axis, main_program=None, startup_program=None): + """ + This function concats the input along the axis mentioned + and returns that as the output. + """ helper = LayerHelper('concat', **locals()) out = helper.create_tmp_variable(dtype=helper.input_dtype()) helper.append_op( @@ -204,13 +446,71 @@ def concat(input, axis, main_program=None, startup_program=None): def sums(input, main_program=None, startup_program=None): + """ + This function takes in the input and performs the sum operation on it + and returns that as the output. + """ helper = LayerHelper('sum', **locals()) out = helper.create_tmp_variable(dtype=helper.input_dtype()) helper.append_op(type='sum', inputs={'X': input}, outputs={'Out': out}) return out +def assign(input, output, main_program=None): + helper = LayerHelper('assign', **locals()) + helper.append_op( + type='scale', + inputs={'X': [input]}, + outputs={'Out': [output]}, + attrs={'scale': 1.0}) + return output + + +def split_lod_tensor(input, + mask, + level, + main_program=None, + startup_program=None): + helper = LayerHelper('split_lod_tensor', **locals()) + out_true = helper.create_tmp_variable(dtype=input.data_type) + out_false = helper.create_tmp_variable(dtype=input.data_type) + helper.append_op( + type='split_lod_tensor', + inputs={ + 'X': input, + 'Mask': mask, + }, + outputs={'OutTrue': out_true, + 'OutFalse': out_false}, + attrs={'level': level}) + return out_true, out_false + + +def merge_lod_tensor(in_true, + in_false, + x, + mask, + level, + main_program=None, + startup_program=None): + helper = LayerHelper('merge_lod_tensor', **locals()) + out = helper.create_tmp_variable(dtype=x.data_type) + helper.append_op( + type='merge_lod_tensor', + inputs={'X': x, + 'Mask': mask, + 'InTrue': in_true, + 'InFalse': in_false}, + outputs={'Out': out}, + attrs={'level': level}) + return out + + def cos_sim(X, Y, **kwargs): + """ + This function performs the cosine similarity between two tensors + X and Y and returns that as the output. + """ helper = LayerHelper('cos_sim', **kwargs) out = helper.create_tmp_variable(dtype=X.data_type) xnorm = helper.create_tmp_variable(dtype=X.data_type) @@ -226,6 +526,9 @@ def cos_sim(X, Y, **kwargs): def cross_entropy(input, label, **kwargs): + """ + This function computes cross_entropy using the input and label. + """ helper = LayerHelper('cross_entropy', **kwargs) out = helper.create_tmp_variable(dtype=input.data_type) helper.append_op( @@ -238,6 +541,10 @@ def cross_entropy(input, label, **kwargs): def square_error_cost(input, label, **kwargs): + """ + This functions returns the squared error cost using the input and label. + The output is appending the op to do the above. + """ helper = LayerHelper('square_error_cost', **kwargs) minus_out = helper.create_tmp_variable(dtype=input.data_type) helper.append_op( @@ -253,6 +560,10 @@ def square_error_cost(input, label, **kwargs): def accuracy(input, label, k=1, **kwargs): + """ + This function computes the accuracy using the input and label. + The output is the top_k inputs and their indices. + """ helper = LayerHelper("accuracy", **kwargs) topk_out = helper.create_tmp_variable(dtype=input.data_type) topk_indices = helper.create_tmp_variable(dtype="int64") @@ -291,6 +602,11 @@ def sequence_conv(input, param_attr=None, main_program=None, startup_program=None): + """ + This function creates the op for sequence_conv, using the inputs and + other convolutional configurations for the filters and stride as given + in the input parameters to the function. + """ # FIXME(dzh) : want to unify the argument of python layer # function. So we ignore some unecessary attributes. # such as, padding_trainable, context_start. @@ -331,6 +647,13 @@ def conv2d(input, param_attr=None, main_program=None, startup_program=None): + """ + This function creates the op for a 2-dimensional Convolution. + This is performed using the parameters of filters(size, dimensionality etc) + , stride and other configurations for a Convolution operation. + This funciton can also append an activation on top of the + conv-2d output, if mentioned in the input parameters. + """ helper = LayerHelper('conv2d', **locals()) dtype = helper.input_dtype() @@ -377,6 +700,11 @@ def conv2d(input, def sequence_pool(input, pool_type, **kwargs): + """ + This function add the operator for sequence pooling. + This is applied on top of the input using pool_type mentioned + in the parameters. + """ helper = LayerHelper('sequence_pool', input=input, **kwargs) dtype = helper.input_dtype() pool_out = helper.create_tmp_variable(dtype) @@ -400,6 +728,10 @@ def pool2d(input, global_pooling=False, main_program=None, startup_program=None): + """ + This function adds the operator for pooling in 2 dimensions, using the + pooling configurations mentioned in input parameters. + """ if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", @@ -420,9 +752,9 @@ def pool2d(input, inputs={"X": input}, outputs={"Out": pool_out}, attrs={ - "poolingType": pool_type, + "pooling_type": pool_type, "ksize": pool_size, - "globalPooling": global_pooling, + "global_pooling": global_pooling, "strides": pool_stride, "paddings": pool_padding }) @@ -440,6 +772,10 @@ def batch_norm(input, data_layout='NCHW', main_program=None, startup_program=None): + """ + This function helps create an operator to implement + the BatchNorm layer using the configurations from the input parameters. + """ helper = LayerHelper('batch_norm', **locals()) dtype = helper.input_dtype() @@ -511,8 +847,10 @@ def batch_norm(input, class BlockGuard(object): """ - BlockGuard used to create sub-block in program by using Python `with` - keyword. + BlockGuard class. + + BlockGuard class is used to create a sub-block in a program by + using the Python `with` keyword. """ def __init__(self, main_program): @@ -531,9 +869,15 @@ def __exit__(self, exc_type, exc_val, exc_tb): class StaticRNNGuard(BlockGuard): + """ + StaticRNNGuard class. + + StaticRNNGuard class is used to create a StaticRNN block in a program. + """ + def __init__(self, rnn): if not isinstance(rnn, StaticRNN): - raise TypeError("StaticRNNGuard takes an StaticRNN") + raise TypeError("StaticRNNGuard takes a StaticRNN") super(StaticRNNGuard, self).__init__(rnn.helper.main_program) self.rnn = rnn @@ -551,12 +895,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): class StaticRNNMemoryLink(object): """ - :param init: the initial variable for Memory - :type init: Variable - :param pre_mem: the memory variable in previous time step - :type pre_mem: Variable - :param mem: the memory variable in current time step - :type mem: Variable + StaticRNNMemoryLink class. + + Args: + init: the initial variable for Memory + init: Variable + pre_mem: the memory variable in previous time step + pre_mem: Variable + mem: the memory variable in current time step + mem: Variable + + StaticRNNMemoryLink class is used to create a link between two + memory cells of a StaticRNN. """ def __init__(self, init, pre_mem, mem=None): @@ -566,6 +916,12 @@ def __init__(self, init, pre_mem, mem=None): class StaticRNN(object): + """ + StaticRNN class. + + StaticRNN class is used to create a StaticRNN. The RNN will have its + own parameters like inputs, outputs, memories, status and length. + """ BEFORE_RNN_BLOCK = 0 IN_RNN_BLOCK = 1 AFTER_RNN_BLOCK = 2 @@ -594,15 +950,15 @@ def memory(self, init_value=0.0, init_batch_dim_idx=0, ref_batch_dim_idx=1): - ''' - :param init: boot memory, if not set, a shape, batch_ref must be provided - :param shape: shape of the boot memory - :param batch_ref: batch size reference variable - :param init_value: the init value of boot memory - :param init_batch_dim_idx: the index of batch size in init's dimension - :param ref_batch_dim_idx: the index of batch size in batch_ref's dimension - :return: boot memory - ''' + """ + Args: + init: boot memory, if not set, a shape, batch_ref must be provided + shape: shape of the boot memory + batch_ref: batch size reference variable + init_value: the init value of boot memory + init_batch_dim_idx: the index of batch size in init's dimension + ref_batch_dim_idx: the index of batch size in batch_ref's dimension + """ self._assert_in_rnn_block_('memory') if init is None: if shape is None or batch_ref is None: @@ -768,7 +1124,131 @@ def complete_rnn_op(self): }) +class WhileGuard(BlockGuard): + def __init__(self, while_op): + if not isinstance(while_op, While): + raise TypeError("WhileGuard takes a while op") + super(WhileGuard, self).__init__(while_op.helper.main_program) + self.while_op = while_op + + def __enter__(self): + self.while_op.status = While.IN_WHILE_BLOCK + return super(WhileGuard, self).__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + return False + self.while_op.status = While.AFTER_WHILE_BLOCK + self.while_op.complete() + return super(WhileGuard, self).__exit__(exc_type, exc_val, exc_tb) + + +class While(object): + BEFORE_WHILE_BLOCK = 0 + IN_WHILE_BLOCK = 1 + AFTER_WHILE_BLOCK = 2 + + def __init__(self, cond, name=None, main_program=None): + self.helper = LayerHelper("while", name=name, main_program=main_program) + self.status = While.BEFORE_WHILE_BLOCK + if not isinstance(cond, Variable): + raise TypeError("condition should be a variable") + assert isinstance(cond, Variable) + if cond.data_type != core.DataType.BOOL: + raise TypeError("condition should be a bool variable") + if reduce(lambda a, b: a * b, cond.shape, 1) != 1: + raise TypeError("condition should be a bool scalar") + self.cond_var = cond + + def block(self): + return WhileGuard(self) + + def complete(self): + main_program = self.helper.main_program + while_block = main_program.current_block() + parent_block = main_program.block(main_program.current_block() + .parent_idx) + + inner_outputs = {self.cond_var.name} + x_name_list = set() + for op in while_block.ops: + for iname in op.input_names: + for in_var_name in op.input(iname): + if in_var_name not in inner_outputs: + x_name_list.add(in_var_name) + + for oname in op.output_names: + for out_var_name in op.output(oname): + inner_outputs.add(out_var_name) + + out_vars = [] + for inner_out_name in inner_outputs: + if inner_out_name in parent_block.vars: + out_vars.append(parent_block.var(inner_out_name)) + + step_scope = parent_block.create_var( + type=core.VarDesc.VarType.STEP_SCOPES) + + parent_block.append_op( + type='while', + inputs={ + 'X': [parent_block.var(x_name) for x_name in x_name_list], + 'Condition': [self.cond_var] + }, + outputs={'Out': out_vars, + 'StepScopes': [step_scope]}, + attrs={'step_block': while_block}) + + +def lstm(x, + c_pre_init, + hidden_dim, + forget_bias=None, + main_program=None, + startup_program=None): + """ + This function helps create an operator for the LSTM (Long Short Term + Memory) cell that can be used inside an RNN. + """ + helper = LayerHelper('lstm_unit', **locals()) + rnn = StaticRNN() + with rnn.step(): + c_pre = rnn.memory(init=c_pre_init) + x_t = rnn.step_input(x) + + before_fc = concat( + input=[x_t, c_pre], + axis=1, + main_program=main_program, + startup_program=startup_program) + after_fc = fc(input=before_fc, + size=hidden_dim * 4, + main_program=main_program, + startup_program=startup_program) + + data_type = x.data_type + c = helper.create_tmp_variable(data_type) + h = helper.create_tmp_variable(data_type) + + helper.append_op( + type='lstm_unit', + inputs={"X": after_fc, + "C_prev": c_pre}, + outputs={"C": c, + "H": h}, + attrs={"forget_bias": forget_bias}) + + rnn.update_memory(c_pre, c) + rnn.output(h) + + return rnn() + + def lod_rank_table(x, level=0, main_program=None): + """ + This function creates an operator for creating a LOD_RANK_TABLE + using the input x. + """ helper = LayerHelper("lod_rank_table", **locals()) table = helper.create_variable( type=core.VarDesc.VarType.LOD_RANK_TABLE, @@ -782,10 +1262,15 @@ def lod_rank_table(x, level=0, main_program=None): def lod_tensor_to_array(x, table, main_program=None): + """ + This function creates an operator to convert an LOD_Tensor to + an array. + """ helper = LayerHelper("lod_tensor_to_array", **locals()) array = helper.create_variable( name=unique_name("lod_tensor_to_array"), - type=core.VarDesc.VarType.LOD_TENSOR_ARRAY) + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + dtype=x.data_type) helper.append_op( type='lod_tensor_to_array', inputs={'X': x, @@ -795,6 +1280,10 @@ def lod_tensor_to_array(x, table, main_program=None): def array_to_lod_tensor(x, table, main_program=None): + """ + This function creates an operator to convert an array to a + LOD_Tensor. + """ helper = LayerHelper("array_to_lod_tensor", **locals()) tmp = helper.create_tmp_variable(dtype=x.data_type) helper.append_op( @@ -806,7 +1295,12 @@ def array_to_lod_tensor(x, table, main_program=None): def fill_constant(shape, dtype, value, main_program=None): - helper = LayerHelper("ones", **locals()) + """ + This function creates a tensor , with shape as mentioned in the input and + specified data_type and fills this up with a constant value that + comes in the input. It also sets the stop_gradient to be True. + """ + helper = LayerHelper("fill_constant", **locals()) out = helper.create_tmp_variable(dtype=dtype) helper.append_op( type='fill_constant', @@ -822,25 +1316,45 @@ def fill_constant(shape, dtype, value, main_program=None): def ones(shape, dtype, main_program=None): + """ + This function performs the same function as fill_constant() declared above + with the constant value being 1.0. + """ return fill_constant(value=1.0, **locals()) def zeros(shape, dtype, main_program=None): + """ + This function performs the same function as fill_constant() declared above + with the constant value being 0.0. + """ return fill_constant(value=0.0, **locals()) -def increment(x, value=1.0, main_program=None): +def increment(x, value=1.0, in_place=True, main_program=None): + """ + This function creates an operator to increment each value in the input + `x` by an amount: `value` as mentioned in the input parameter. This + operation is performed in-place by default. + """ helper = LayerHelper("increment", **locals()) - tmp = helper.create_tmp_variable(dtype=x.data_type) + if not in_place: + out = helper.create_tmp_variable(dtype=x.data_type) + else: + out = x helper.append_op( type='increment', inputs={'X': [x]}, - outputs={'Out': [tmp]}, + outputs={'Out': [out]}, attrs={'step': value}) - return tmp + return out def array_write(x, i, array=None, main_program=None): + """ + This function creates an operator to write the data out as a + LOD_TENSOR_ARRAY. + """ helper = LayerHelper('array_write', **locals()) if array is None: array = helper.create_variable( @@ -855,7 +1369,31 @@ def array_write(x, i, array=None, main_program=None): return array +def create_array(dtype, main_program=None): + helper = LayerHelper("array", **locals()) + return helper.create_variable( + name="{0}.out".format(helper.name), + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + dtype=dtype) + + +def less_than(x, y, cond=None, main_program=None): + helper = LayerHelper("less_than", **locals()) + if cond is None: + cond = helper.create_tmp_variable(dtype='bool') + cond.stop_gradient = True + + helper.append_op( + type='less_than', inputs={'X': [x], + 'Y': [y]}, outputs={'Out': [cond]}) + return cond + + def array_read(array, i, main_program=None): + """ + This function creates an operator to read the data in as a + LOD_TENSOR_ARRAY. + """ helper = LayerHelper('array_read', **locals()) if not isinstance( array, @@ -868,3 +1406,103 @@ def array_read(array, i, main_program=None): 'I': [i]}, outputs={'Out': [out]}) return out + + +def shrink_memory(x, i, table, main_program=None): + """ + This function creates an operator to shrink_rnn_memory using the RankTable + as mentioned in the input parameter. + """ + helper = LayerHelper('shrink_memory', **locals()) + out = helper.create_tmp_variable(dtype=x.data_type) + helper.append_op( + type='shrink_rnn_memory', + inputs={'X': [x], + 'I': [i], + 'RankTable': [table]}, + outputs={'Out': [out]}, + attrs={}) + return out + + +def array_length(array, main_program=None): + """ + This function creates an operator to find the length of the + LOD_TENSOR_ARRAY. + """ + helper = LayerHelper('array_length', **locals()) + tmp = helper.create_tmp_variable(dtype='int64') + tmp.stop_gradient = True + helper.append_op( + type='lod_array_length', inputs={'X': [array]}, outputs={'Out': [tmp]}) + return tmp + + +class ConditionalBlockGuard(BlockGuard): + def __init__(self, block): + if not isinstance(block, ConditionalBlock): + raise TypeError("block should be conditional block") + super(ConditionalBlockGuard, self).__init__(block.helper.main_program) + self.block = block + + def __enter__(self): + return super(ConditionalBlockGuard, self).__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.block.complete() + return super(ConditionalBlockGuard, self).__exit__(exc_type, exc_val, + exc_tb) + + +class ConditionalBlock(object): + def __init__(self, inputs, name=None, main_program=None): + for each_input in inputs: + if not isinstance(each_input, Variable): + raise TypeError("Each input should be variable") + self.inputs = inputs + self.helper = LayerHelper( + 'conditional_block', name=name, main_program=main_program) + + def block(self): + return ConditionalBlockGuard(self) + + def complete(self): + inside_block = self.helper.main_program.current_block() + parent_block = self.helper.main_program.block(inside_block.parent_idx) + + intermediate = set() + params = set() + + for each_op in inside_block.ops: + assert isinstance(each_op, Operator) + for iname in each_op.input_names: + for in_var_name in each_op.input(iname): + if in_var_name not in intermediate: + params.add(in_var_name) + + for oname in each_op.output_names: + for out_var_name in each_op.output(oname): + intermediate.add(out_var_name) + input_set = set([ipt.name for ipt in self.inputs]) + + param_list = [ + parent_block.var(each_name) for each_name in params + if each_name not in input_set + ] + + out_list = [ + parent_block.var(var_name) for var_name in parent_block.vars + if var_name not in intermediate + ] + + step_scope = parent_block.create_var( + type=core.VarDesc.VarType.STEP_SCOPES) + parent_block.append_op( + type='conditional_block', + inputs={ + 'X': self.inputs, + 'Params': param_list, + }, + outputs={'Out': out_list, + 'Scope': [step_scope]}, + attrs={'block': inside_block}) diff --git a/python/paddle/v2/framework/optimizer.py b/python/paddle/v2/framework/optimizer.py index f20865d604f68..f06c0fb98d572 100644 --- a/python/paddle/v2/framework/optimizer.py +++ b/python/paddle/v2/framework/optimizer.py @@ -35,15 +35,21 @@ def _append_optimize_op(self, block, param_and_grad): """ raise NotImplementedError() - def _initialize_tensors(self, block): - """Create all necessary tensors, that will be shared for all parameter updates. - - Tensors like learning rate should be initialized here. - - Args: - block: the block in which the loss variable is present - """ - pass + def _create_param_lr(self, param_and_grad): + # create learning rate variable for every parameter + param = param_and_grad[0] + param_lr = param.optimize_attr['learning_rate'] + param_lr_shape = [1] + param_lr_var = self.helper.create_global_variable( + name=unique_name("learning_rate"), + dtype='float32', + shape=param_lr_shape, + lod_level=1, + persistable=True) + param_lr = param_lr * self._learning_rate + self.helper.set_variable_initializer( + var=param_lr_var, initializer=ConstantInitializer(param_lr)) + return param_lr_var def _create_accumulators(self, block, parameters): """Create all accumulators needed by the parameters @@ -161,8 +167,6 @@ def create_optimization_pass(self, startup_program=startup_program) self._create_accumulators(loss.block, [p[0] for p in parameters_and_grads]) - # Create any necessary tensors - self._initialize_tensors(loss.block) optimize_ops = [] for param_and_grad in parameters_and_grads: @@ -214,27 +218,16 @@ def __init__(self, learning_rate, global_step=None): self.type = "sgd" self._learning_rate = learning_rate - def _initialize_tensors(self, block): - lr_shape = [1] - # create a variable for learning_rate - self._lr = self.helper.create_global_variable( - name=unique_name("learning_rate"), - dtype='float32', - shape=lr_shape, - lod_level=1, - persistable=True) - self.helper.set_variable_initializer( - var=self._lr, initializer=ConstantInitializer(self._learning_rate)) - def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) + # create the optimize op sgd_op = block.append_op( type=self.type, inputs={ "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": self._lr + "LearningRate": self._create_param_lr(param_and_grad) }, outputs={"ParamOut": param_and_grad[0]}) @@ -259,19 +252,6 @@ def __init__(self, self._momentum = momentum self._use_nesterov = bool(use_nesterov) - def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) - lr_shape = [1] - # create a variable for learning_rate - self._lr = self.helper.create_global_variable( - name=unique_name("learning_rate"), - dtype='float32', - shape=lr_shape, - lod_level=1, - persistable=True) - self.helper.set_variable_initializer( - var=self._lr, initializer=ConstantInitializer(self._learning_rate)) - def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -290,14 +270,14 @@ def _append_optimize_op(self, block, param_and_grad): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Velocity": velocity_acc, - "LearningRate": self._lr + "LearningRate": self._create_param_lr(param_and_grad) }, outputs={ "ParamOut": param_and_grad[0], "VelocityOut": velocity_acc }, attrs={"mu": self._momentum, - "useNesterov": self._use_nesterov}) + "use_nesterov": self._use_nesterov}) return momentum_op @@ -315,18 +295,6 @@ def __init__(self, learning_rate, epsilon=1.0e-6, global_step=None): self._learning_rate = learning_rate self._epsilon = epsilon - def _initialize_tensors(self, block): - lr_shape = [1] - # create a variable for learning_rate - self._lr = self.helper.create_global_variable( - name=unique_name("learning_rate"), - dtype='float32', - shape=lr_shape, - lod_level=1, - persistable=True) - self.helper.set_variable_initializer( - var=self._lr, initializer=ConstantInitializer(self._learning_rate)) - def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -346,7 +314,7 @@ def _append_optimize_op(self, block, param_and_grad): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Moment": moment_acc, - "LearningRate": self._lr + "LearningRate": self._create_param_lr(param_and_grad) }, outputs={"ParamOut": param_and_grad[0], "MomentOut": moment_acc}, @@ -378,18 +346,6 @@ def __init__(self, self._beta2 = beta2 self._epsilon = epsilon - def _initialize_tensors(self, block): - lr_shape = [1] - # create a variable for learning_rate - self._lr = self.helper.create_global_variable( - name=unique_name("learning_rate"), - dtype='float32', - shape=lr_shape, - lod_level=1, - persistable=True) - self.helper.set_variable_initializer( - var=self._lr, initializer=ConstantInitializer(self._learning_rate)) - def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -433,7 +389,7 @@ def _append_optimize_op(self, block, param_and_grad): inputs={ "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": self._lr, + "LearningRate": self._create_param_lr(param_and_grad), "Moment1": moment1, "Moment2": moment2, "Beta1Pow": self._beta1_pow_acc, @@ -495,18 +451,6 @@ def __init__(self, self._beta2 = beta2 self._epsilon = epsilon - def _initialize_tensors(self, block): - lr_shape = [1] - # create a variable for learning_rate - self._lr = self.helper.create_global_variable( - name=unique_name("learning_rate"), - dtype='float32', - shape=lr_shape, - lod_level=1, - persistable=True) - self.helper.set_variable_initializer( - var=self._lr, initializer=ConstantInitializer(self._learning_rate)) - def _create_accumulators(self, block, parameters): # Create beta1 power accumulator tensor beta_shape = [1] @@ -536,7 +480,7 @@ def _append_optimize_op(self, block, param_and_grad): inputs={ "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": self._lr, + "LearningRate": self._create_param_lr(param_and_grad), "Moment": moment, "InfNorm": inf_norm, "Beta1Pow": self._beta1_pow_acc diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt index 4d7664469e481..e795627bfe9e8 100644 --- a/python/paddle/v2/framework/tests/CMakeLists.txt +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -3,3 +3,5 @@ string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") foreach(src ${TEST_OPS}) py_test(${src} SRCS ${src}.py) endforeach() + +add_subdirectory(book) diff --git a/python/paddle/v2/framework/tests/book/CMakeLists.txt b/python/paddle/v2/framework/tests/book/CMakeLists.txt new file mode 100644 index 0000000000000..4d7664469e481 --- /dev/null +++ b/python/paddle/v2/framework/tests/book/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") +string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") +foreach(src ${TEST_OPS}) + py_test(${src} SRCS ${src}.py) +endforeach() diff --git a/python/paddle/v2/framework/tests/test_fit_a_line.py b/python/paddle/v2/framework/tests/book/test_fit_a_line.py similarity index 97% rename from python/paddle/v2/framework/tests/test_fit_a_line.py rename to python/paddle/v2/framework/tests/book/test_fit_a_line.py index 174ee74c3bc89..6e09b88dca34d 100644 --- a/python/paddle/v2/framework/tests/test_fit_a_line.py +++ b/python/paddle/v2/framework/tests/book/test_fit_a_line.py @@ -3,7 +3,7 @@ import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_main_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.io import save_persistables, load_persistables from paddle.v2.framework.executor import Executor diff --git a/python/paddle/v2/framework/tests/test_image_classification_train.py b/python/paddle/v2/framework/tests/book/test_image_classification_train.py similarity index 100% rename from python/paddle/v2/framework/tests/test_image_classification_train.py rename to python/paddle/v2/framework/tests/book/test_image_classification_train.py diff --git a/python/paddle/v2/framework/tests/test_recognize_digits_conv.py b/python/paddle/v2/framework/tests/book/test_recognize_digits_conv.py similarity index 97% rename from python/paddle/v2/framework/tests/test_recognize_digits_conv.py rename to python/paddle/v2/framework/tests/book/test_recognize_digits_conv.py index 9ec45814a09e6..111005e5bcc42 100644 --- a/python/paddle/v2/framework/tests/test_recognize_digits_conv.py +++ b/python/paddle/v2/framework/tests/book/test_recognize_digits_conv.py @@ -5,7 +5,7 @@ import paddle.v2.framework.optimizer as optimizer import paddle.v2.framework.evaluator as evaluator -from paddle.v2.framework.framework import Program, g_main_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.executor import Executor import numpy as np diff --git a/python/paddle/v2/framework/tests/test_recognize_digits_mlp.py b/python/paddle/v2/framework/tests/book/test_recognize_digits_mlp.py similarity index 100% rename from python/paddle/v2/framework/tests/test_recognize_digits_mlp.py rename to python/paddle/v2/framework/tests/book/test_recognize_digits_mlp.py diff --git a/python/paddle/v2/framework/tests/test_recommender_system.py b/python/paddle/v2/framework/tests/book/test_recommender_system.py similarity index 99% rename from python/paddle/v2/framework/tests/test_recommender_system.py rename to python/paddle/v2/framework/tests/book/test_recommender_system.py index 7e54f0d1b8646..31562b4391d16 100644 --- a/python/paddle/v2/framework/tests/test_recommender_system.py +++ b/python/paddle/v2/framework/tests/book/test_recommender_system.py @@ -4,7 +4,7 @@ import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_main_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.executor import Executor import numpy as np diff --git a/python/paddle/v2/framework/tests/test_understand_sentiment_conv.py b/python/paddle/v2/framework/tests/book/test_understand_sentiment_conv.py similarity index 100% rename from python/paddle/v2/framework/tests/test_understand_sentiment_conv.py rename to python/paddle/v2/framework/tests/book/test_understand_sentiment_conv.py diff --git a/python/paddle/v2/framework/tests/book/test_understand_sentiment_dynamic_lstm.py b/python/paddle/v2/framework/tests/book/test_understand_sentiment_dynamic_lstm.py new file mode 100644 index 0000000000000..2457c71e1a627 --- /dev/null +++ b/python/paddle/v2/framework/tests/book/test_understand_sentiment_dynamic_lstm.py @@ -0,0 +1,110 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.nets as nets +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import Program, g_main_program, g_startup_program +from paddle.v2.framework.executor import Executor + +import numpy as np + + +def stacked_lstm_net(input_dim, + class_dim=2, + emb_dim=128, + hid_dim=512, + stacked_num=3): + assert stacked_num % 2 == 1 + data = layers.data(name="words", shape=[1], data_type="int64") + label = layers.data(name="label", shape=[1], data_type="int64") + + emb = layers.embedding(input=data, size=[input_dim, emb_dim]) + # add bias attr + + # TODO(qijun) linear act + fc1 = layers.fc(input=emb, size=hid_dim) + lstm1, cell1 = layers.dynamic_lstm(input=fc1, size=hid_dim) + + inputs = [fc1, lstm1] + + for i in range(2, stacked_num + 1): + fc = layers.fc(input=inputs, size=hid_dim) + lstm, cell = layers.dynamic_lstm( + input=fc, size=hid_dim, is_reverse=(i % 2) == 0) + inputs = [fc, lstm] + + fc_last = layers.sequence_pool(input=inputs[0], pool_type='max') + lstm_last = layers.sequence_pool(input=inputs[1], pool_type='max') + + prediction = layers.fc(input=[fc_last, lstm_last], + size=class_dim, + act='softmax') + cost = layers.cross_entropy(input=prediction, label=label) + avg_cost = layers.mean(x=cost) + adam_optimizer = optimizer.AdamOptimizer(learning_rate=0.002) + opts = adam_optimizer.minimize(avg_cost) + acc = layers.accuracy(input=prediction, label=label) + return avg_cost, acc + + +def to_lodtensor(data, place): + seq_lens = [len(seq) for seq in data] + cur_len = 0 + lod = [cur_len] + for l in seq_lens: + cur_len += l + lod.append(cur_len) + flattened_data = np.concatenate(data, axis=0).astype("int64") + flattened_data = flattened_data.reshape([len(flattened_data), 1]) + res = core.LoDTensor() + res.set(flattened_data, place) + res.set_lod([lod]) + return res + + +def main(): + BATCH_SIZE = 100 + PASS_NUM = 5 + + word_dict = paddle.dataset.imdb.word_dict() + print "load word dict successfully" + dict_dim = len(word_dict) + class_dim = 2 + + cost, acc = stacked_lstm_net(input_dim=dict_dim, class_dim=class_dim) + + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=1000), + batch_size=BATCH_SIZE) + place = core.CPUPlace() + exe = Executor(place) + + exe.run(g_startup_program) + + for pass_id in xrange(PASS_NUM): + for data in train_data(): + tensor_words = to_lodtensor(map(lambda x: x[0], data), place) + + label = np.array(map(lambda x: x[1], data)).astype("int64") + label = label.reshape([BATCH_SIZE, 1]) + + tensor_label = core.LoDTensor() + tensor_label.set(label, place) + + outs = exe.run(g_main_program, + feed={"words": tensor_words, + "label": tensor_label}, + fetch_list=[cost, acc]) + cost_val = np.array(outs[0]) + acc_val = np.array(outs[1]) + + print("cost=" + str(cost_val) + " acc=" + str(acc_val)) + if cost_val < 1.0 and acc_val > 0.7: + exit(0) + exit(1) + + +if __name__ == '__main__': + main() diff --git a/python/paddle/v2/framework/tests/book/test_understand_sentiment_lstm.py b/python/paddle/v2/framework/tests/book/test_understand_sentiment_lstm.py new file mode 100644 index 0000000000000..26cbd01bc0491 --- /dev/null +++ b/python/paddle/v2/framework/tests/book/test_understand_sentiment_lstm.py @@ -0,0 +1,107 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import g_main_program, g_startup_program +from paddle.v2.framework.executor import Executor + +import numpy as np + + +def lstm_net(dict_dim, class_dim=2, emb_dim=32, seq_len=80, batch_size=50): + data = layers.data( + name="words", + shape=[seq_len * batch_size, 1], + append_batch_size=False, + data_type="int64") + label = layers.data( + name="label", + shape=[batch_size, 1], + append_batch_size=False, + data_type="int64") + + emb = layers.embedding(input=data, size=[dict_dim, emb_dim]) + emb = layers.reshape(x=emb, shape=[batch_size, seq_len, emb_dim]) + emb = layers.transpose(x=emb, axis=[1, 0, 2]) + + c_pre_init = layers.fill_constant( + dtype=emb.data_type, shape=[batch_size, emb_dim], value=0.0) + layer_1_out = layers.lstm(emb, c_pre_init=c_pre_init, hidden_dim=emb_dim) + layer_1_out = layers.transpose(x=layer_1_out, axis=[1, 0, 2]) + + prediction = layers.fc(input=layer_1_out, size=class_dim, act="softmax") + cost = layers.cross_entropy(input=prediction, label=label) + + avg_cost = layers.mean(x=cost) + adam_optimizer = optimizer.AdamOptimizer(learning_rate=0.002) + opts = adam_optimizer.minimize(avg_cost) + acc = layers.accuracy(input=prediction, label=label) + + return avg_cost, acc + + +def to_lodtensor(data, place): + seq_lens = [len(seq) for seq in data] + cur_len = 0 + lod = [cur_len] + for l in seq_lens: + cur_len += l + lod.append(cur_len) + flattened_data = np.concatenate(data, axis=0).astype("int64") + flattened_data = flattened_data.reshape([len(flattened_data), 1]) + res = core.LoDTensor() + res.set(flattened_data, place) + res.set_lod([lod]) + return res + + +def chop_data(data, chop_len=80, batch_len=50): + data = [(x[0][:chop_len], x[1]) for x in data if len(x[0]) >= chop_len] + + return data[:batch_len] + + +def prepare_feed_data(data, place): + tensor_words = to_lodtensor(map(lambda x: x[0], data), place) + + label = np.array(map(lambda x: x[1], data)).astype("int64") + label = label.reshape([50, 1]) + tensor_label = core.LoDTensor() + tensor_label.set(label, place) + + return tensor_words, tensor_label + + +def main(): + word_dict = paddle.dataset.imdb.word_dict() + cost, acc = lstm_net(dict_dim=len(word_dict), class_dim=2) + + batch_size = 100 + train_data = paddle.batch( + paddle.reader.buffered( + paddle.dataset.imdb.train(word_dict), size=batch_size * 10), + batch_size=batch_size) + + data = chop_data(next(train_data())) + + place = core.CPUPlace() + tensor_words, tensor_label = prepare_feed_data(data, place) + exe = Executor(place) + exe.run(g_startup_program) + + while True: + outs = exe.run(g_main_program, + feed={"words": tensor_words, + "label": tensor_label}, + fetch_list=[cost, acc]) + cost_val = np.array(outs[0]) + acc_val = np.array(outs[1]) + + print("cost=" + str(cost_val) + " acc=" + str(acc_val)) + if acc_val > 0.9: + break + + +if __name__ == '__main__': + main() diff --git a/python/paddle/v2/framework/tests/test_word2vec.py b/python/paddle/v2/framework/tests/book/test_word2vec.py similarity index 98% rename from python/paddle/v2/framework/tests/test_word2vec.py rename to python/paddle/v2/framework/tests/book/test_word2vec.py index 116854c97b373..cb9fc2ab62b56 100644 --- a/python/paddle/v2/framework/tests/test_word2vec.py +++ b/python/paddle/v2/framework/tests/book/test_word2vec.py @@ -3,7 +3,7 @@ import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_main_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.executor import Executor import numpy as np diff --git a/python/paddle/v2/framework/tests/op_test.py b/python/paddle/v2/framework/tests/op_test.py index 2e6710b5fcfe5..4a269341a4be6 100644 --- a/python/paddle/v2/framework/tests/op_test.py +++ b/python/paddle/v2/framework/tests/op_test.py @@ -215,7 +215,11 @@ def feed_var(self, input_vars, place): if isinstance(input_vars[var_name], list): for name, np_value in self.inputs[var_name]: tensor = core.LoDTensor() - tensor.set(np_value, place) + if isinstance(np_value, tuple): + tensor.set(np_value[0], place) + tensor.set_lod(np_value[1]) + else: + tensor.set(np_value, place) feed_map[name] = tensor else: tensor = core.LoDTensor() @@ -236,7 +240,6 @@ def check_output_with_place(self, place, atol): inputs = append_input_output(block, op_proto, self.inputs, True) outputs = append_input_output(block, op_proto, self.outputs, False) - op = block.append_op( type=self.op_type, inputs=inputs, @@ -397,9 +400,11 @@ def _create_var_descs_(block, var_dict): if not isinstance(item[0], basestring): item = [[param_name] + list(item)] if len(item) == 2: - # only set var name and value, set lod to None - var[i] = list(item) + [None] - + if isinstance(item[1], tuple): + var[i] = [item[0], item[1][0], item[1][1]] + else: + # only set var name and value, set lod to None + var[i] = list(item) + [None] var_descs = [(block.create_var( name=name, shape=each.shape, dtype=each.dtype), each, lod) for name, each, lod in var] diff --git a/python/paddle/v2/framework/tests/test_array_read_write_op.py b/python/paddle/v2/framework/tests/test_array_read_write_op.py index b2a2ff2b82133..79e9938216e2a 100644 --- a/python/paddle/v2/framework/tests/test_array_read_write_op.py +++ b/python/paddle/v2/framework/tests/test_array_read_write_op.py @@ -20,21 +20,19 @@ def test_read_write(self): each_x.stop_gradient = False i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = False arr = layers.array_write(x=x[0], i=i) i = layers.increment(x=i) - i.stop_gradient = True arr = layers.array_write(x=x[1], i=i, array=arr) i = layers.increment(x=i) - i.stop_gradient = True arr = layers.array_write(x=x[2], i=i, array=arr) i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = False a0 = layers.array_read(array=arr, i=i) i = layers.increment(x=i) - i.stop_gradient = True # index should not calculate gradient a1 = layers.array_read(array=arr, i=i) i = layers.increment(x=i) - i.stop_gradient = True a2 = layers.array_read(array=arr, i=i) mean_a0 = layers.mean(x=a0) diff --git a/python/paddle/v2/framework/tests/test_assign_op.py b/python/paddle/v2/framework/tests/test_assign_op.py new file mode 100644 index 0000000000000..1b0c145f1a696 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_assign_op.py @@ -0,0 +1,21 @@ +import op_test +import numpy +import unittest + + +class TestAssignOp(op_test.OpTest): + def setUp(self): + self.op_type = "assign" + x = numpy.random.random(size=(100, 10)) + self.inputs = {'X': x} + self.outputs = {'Out': x} + + def test_forward(self): + self.check_output() + + def test_backward(self): + self.check_grad(['X'], 'Out') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_bilinear_tensor_product_op.py b/python/paddle/v2/framework/tests/test_bilinear_tensor_product_op.py new file mode 100644 index 0000000000000..080ca43b8269e --- /dev/null +++ b/python/paddle/v2/framework/tests/test_bilinear_tensor_product_op.py @@ -0,0 +1,37 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestBilinearTensorProductOp(OpTest): + def setUp(self): + self.op_type = "bilinear_tensor_product" + batch_size = 6 + size0 = 3 + size1 = 4 + size2 = 5 + a = np.random.random((batch_size, size0)).astype("float32") + b = np.random.random((batch_size, size1)).astype("float32") + w = np.random.random((size2, size0, size1)).astype("float32") + bias = np.random.random((1, size2)).astype("float32") + output = np.zeros((batch_size, size2)).astype("float32") + for i in range(size2): + w_i = w[i, :, :] + output[:, i] = np.sum(np.matmul(a, w_i) * b, axis=1) + self.inputs = { + 'X': a, + 'Y': b, + 'Weight': w, + 'Bias': bias, + } + self.outputs = {'Out': output + bias} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y', 'Weight', 'Bias'], 'Out') + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_chunk_eval_op.py b/python/paddle/v2/framework/tests/test_chunk_eval_op.py new file mode 100644 index 0000000000000..48673296a6771 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_chunk_eval_op.py @@ -0,0 +1,179 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class Segment(object): + def __init__(self, chunk_type, start_idx, end_idx): + self.chunk_type = chunk_type + self.start_idx = start_idx + self.end_idx = end_idx + + def __str__(self): + return '(Segment: %s, %s, %s)' % (self.chunk_type, self.start_idx, + self.end_idx) + + __repr__ = __str__ + + +class TestChunkEvalOp(OpTest): + num_sequences = 5 + batch_size = 50 + + def parse_scheme(self): + if self.scheme == 'IOB': + self.num_tag_types = 2 + elif self.scheme == 'IOE': + self.num_tag_types = 2 + + def fill_with_chunks(self, data, chunks): + for chunk in chunks: + if self.scheme == 'IOB': + data[chunk.start_idx] = chunk.chunk_type * self.num_tag_types + data[chunk.start_idx + 1: + chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + self.num_tag_types - 1) + data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + self.num_tag_types - 1 + ) if chunk.start_idx < chunk.end_idx else data[chunk.start_idx] + elif self.scheme == 'IOE': + data[chunk.start_idx: + chunk.end_idx] = chunk.chunk_type * self.num_tag_types + data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + self.num_tag_types - 1) + + def rand_chunks(self, starts, num_chunks): + if num_chunks < 0: + num_chunks = np.random.randint(starts[-1]) + chunks = [] + # generate chunk beginnings + chunk_begins = sorted( + np.random.choice( + range(starts[-1]), num_chunks, replace=False)) + seq_chunk_begins = [] + begin_idx = 0 + # divide chunks into sequences + for i in range(len(starts) - 1): + tmp_chunk_begins = [] + while begin_idx < len(chunk_begins) and chunk_begins[ + begin_idx] < starts[i + 1]: + tmp_chunk_begins.append(chunk_begins[begin_idx]) + begin_idx += 1 + seq_chunk_begins.append(tmp_chunk_begins) + # generate chunk ends + chunk_ends = [] + for i in range(len(seq_chunk_begins)): + for j in range(len(seq_chunk_begins[i])): + low = seq_chunk_begins[i][j] + high = seq_chunk_begins[i][j + 1] if j < len(seq_chunk_begins[ + i]) - 1 else starts[i + 1] + chunk_ends.append(np.random.randint(low, high)) + # generate chunks + for chunk_pos in zip(chunk_begins, chunk_ends): + chunk_type = np.random.randint(self.num_chunk_types) + chunks.append(Segment(chunk_type, *chunk_pos)) + return chunks + + def gen_chunks(self, infer, label, starts): + chunks = self.rand_chunks(starts, + self.num_infer_chunks + self.num_label_chunks + - self.num_correct_chunks) + correct_chunks = np.random.choice( + range(len(chunks)), self.num_correct_chunks, replace=False) + infer_chunks = np.random.choice( + [x for x in range(len(chunks)) if x not in correct_chunks], + self.num_infer_chunks - self.num_correct_chunks, + replace=False) + infer_chunks = sorted(correct_chunks.tolist() + infer_chunks.tolist()) + label_chunks = np.random.choice( + [x for x in range(len(chunks)) if x not in infer_chunks], + self.num_label_chunks - self.num_correct_chunks, + replace=False) + label_chunks = sorted(correct_chunks.tolist() + label_chunks.tolist()) + self.fill_with_chunks(infer, [chunks[idx] for idx in infer_chunks]) + self.fill_with_chunks(label, [chunks[idx] for idx in label_chunks]) + # exclude types in excluded_chunk_types + if len(self.excluded_chunk_types) > 0: + for idx in correct_chunks: + if chunks[idx].chunk_type in self.excluded_chunk_types: + self.num_correct_chunks -= 1 + for idx in infer_chunks: + if chunks[idx].chunk_type in self.excluded_chunk_types: + self.num_infer_chunks -= 1 + for idx in label_chunks: + if chunks[idx].chunk_type in self.excluded_chunk_types: + self.num_label_chunks -= 1 + return self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks + + def set_confs(self): + # Use the IOB scheme and labels with 2 chunk types + self.scheme = 'IOB' + self.num_chunk_types = 2 + self.excluded_chunk_types = [] + self.other_chunk_type = self.num_chunk_types + self.attrs = { + 'num_chunk_types': self.num_chunk_types, + 'chunk_scheme': self.scheme, + 'excluded_chunk_types': self.excluded_chunk_types + } + self.parse_scheme() + self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 4, 5, 9 + + def set_data(self): + infer = np.zeros((self.batch_size, )).astype('int32') + infer.fill(self.num_chunk_types * self.num_tag_types) + label = np.copy(infer) + starts = np.random.choice( + range(1, self.batch_size), self.num_sequences - 1, + replace=False).tolist() + starts.extend([0, self.batch_size]) + starts = sorted(starts) + self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = self.gen_chunks( + infer, label, starts) + self.inputs = { + 'Inference': (infer, [starts]), + 'Label': (label, [starts]) + } + precision = float( + self.num_correct_chunks + ) / self.num_infer_chunks if self.num_infer_chunks else 0 + recall = float(self.num_correct_chunks + ) / self.num_label_chunks if self.num_label_chunks else 0 + f1 = float(2 * precision * recall) / ( + precision + recall) if self.num_correct_chunks else 0 + self.outputs = { + 'Precision': np.asarray( + [precision], dtype='float32'), + 'Recall': np.asarray( + [recall], dtype='float32'), + 'F1-Score': np.asarray( + [f1], dtype='float32') + } + + def setUp(self): + self.op_type = 'chunk_eval' + self.set_confs() + self.set_data() + + def test_check_output(self): + self.check_output() + + +class TestChunkEvalOpWithExclude(TestChunkEvalOp): + def set_confs(self): + # Use the IOE scheme and labels with 3 chunk types + self.scheme = 'IOE' + self.num_chunk_types = 3 + self.excluded_chunk_types = [1] + self.other_chunk_type = self.num_chunk_types + self.attrs = { + 'num_chunk_types': self.num_chunk_types, + 'chunk_scheme': self.scheme, + 'excluded_chunk_types': self.excluded_chunk_types + } + self.parse_scheme() + self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 15, 18, 20 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_clip_by_norm_op.py b/python/paddle/v2/framework/tests/test_clip_by_norm_op.py new file mode 100644 index 0000000000000..02f6108a3a661 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_clip_by_norm_op.py @@ -0,0 +1,50 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestClipByNormOp(OpTest): + def setUp(self): + self.max_relative_error = 0.006 + self.initTestCase() + input = np.random.random(self.shape).astype("float32") + input[np.abs(input) < self.max_relative_error] = 0.5 + self.op_type = "clip_by_norm" + self.inputs = {'X': input, } + self.attrs = {} + self.attrs['max_norm'] = self.max_norm + norm = np.sqrt(np.sum(np.square(input))) + if norm > self.max_norm: + output = self.max_norm * input / norm + else: + output = input + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def initTestCase(self): + self.shape = (100, ) + self.max_norm = 1.0 + + +class TestCase1(TestClipByNormOp): + def initTestCase(self): + self.shape = (100, ) + self.max_norm = 1e20 + + +class TestCase2(TestClipByNormOp): + def initTestCase(self): + self.shape = (16, 16) + self.max_norm = 0.1 + + +class TestCase3(TestClipByNormOp): + def initTestCase(self): + self.shape = (4, 8, 16) + self.max_norm = 1.0 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conditional_block.py b/python/paddle/v2/framework/tests/test_conditional_block.py new file mode 100644 index 0000000000000..9b96ff306c37a --- /dev/null +++ b/python/paddle/v2/framework/tests/test_conditional_block.py @@ -0,0 +1,40 @@ +import unittest +import paddle.v2.framework.layers as layers +import paddle.v2.framework.core as core +from paddle.v2.framework.framework import g_startup_program, g_main_program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops +import numpy + + +class ConditionalBlock(unittest.TestCase): + def test_forward(self): + data = layers.data(name='X', shape=[1], data_type='float32') + data.stop_gradient = False + cond = layers.ConditionalBlock(inputs=[data]) + out = layers.create_tensor(dtype='float32') + with cond.block(): + hidden = layers.fc(input=data, size=10) + layers.assign(hidden, out) + + cpu = core.CPUPlace() + exe = Executor(cpu) + exe.run(g_startup_program) + + x = core.LoDTensor() + x.set(numpy.random.random(size=(10, 1)).astype('float32'), cpu) + + outs = map(numpy.array, exe.run(feed={'X': x}, fetch_list=[out]))[0] + print outs + loss = layers.mean(x=out) + append_backward_ops(loss=loss) + outs = map(numpy.array, + exe.run(feed={'X': x}, + fetch_list=[ + g_main_program.block(0).var(data.name + "@GRAD") + ]))[0] + print outs + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_create_op_doc_string.py b/python/paddle/v2/framework/tests/test_create_op_doc_string.py new file mode 100644 index 0000000000000..d21e96df2a64d --- /dev/null +++ b/python/paddle/v2/framework/tests/test_create_op_doc_string.py @@ -0,0 +1,11 @@ +import unittest +import paddle.v2.framework.layers as layers + + +class TestDocString(unittest.TestCase): + def test_layer_doc_string(self): + print layers.dropout.__doc__ + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_expand_op.py b/python/paddle/v2/framework/tests/test_expand_op.py new file mode 100644 index 0000000000000..0440f7a2bb159 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_expand_op.py @@ -0,0 +1,97 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestExpandOpRank1(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random(12).astype("float32")} + self.attrs = {'expand_times': [2]} + output = np.tile(self.inputs['X'], 2) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank2_Corner(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((12, 14)).astype("float32")} + self.attrs = {'expand_times': [1, 1]} + output = np.tile(self.inputs['X'], (1, 1)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank2(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((12, 14)).astype("float32")} + self.attrs = {'expand_times': [2, 3]} + output = np.tile(self.inputs['X'], (2, 3)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank3_Corner(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((2, 4, 5)).astype("float32")} + self.attrs = {'expand_times': [1, 1, 1]} + output = np.tile(self.inputs['X'], (1, 1, 1)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank3(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((2, 4, 5)).astype("float32")} + self.attrs = {'expand_times': [2, 1, 4]} + output = np.tile(self.inputs['X'], (2, 1, 4)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank4(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((2, 4, 5, 7)).astype("float32")} + self.attrs = {'expand_times': [3, 2, 1, 2]} + output = np.tile(self.inputs['X'], (3, 2, 1, 2)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_increment_op.py b/python/paddle/v2/framework/tests/test_increment_op.py deleted file mode 100644 index e174272b05b94..0000000000000 --- a/python/paddle/v2/framework/tests/test_increment_op.py +++ /dev/null @@ -1,41 +0,0 @@ -import unittest -import numpy as np -from op_test import OpTest - - -class TestIncrementOpPositiveStep(OpTest): - """Test increment op with positive step - """ - - def setUp(self): - self.op_type = "increment" - self.inputs = {'X': np.random.random((10, 10)).astype("float32")} - self.attrs = {'step': 14.8} - self.outputs = {'Out': self.inputs['X'] + self.attrs['step']} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(['X'], 'Out') - - -class TestIncrementOpNegativeStep(OpTest): - """Test increment op with negative step - """ - - def setUp(self): - self.op_type = "increment" - self.inputs = {'X': np.random.random((10, 10)).astype("float32")} - self.attrs = {'step': -3.8} - self.outputs = {'Out': self.inputs['X'] + self.attrs['step']} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(['X'], 'Out') - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_inference_model_io.py b/python/paddle/v2/framework/tests/test_inference_model_io.py index d273387a35820..48984f86a1864 100644 --- a/python/paddle/v2/framework/tests/test_inference_model_io.py +++ b/python/paddle/v2/framework/tests/test_inference_model_io.py @@ -3,7 +3,7 @@ import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_main_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.io import save_inference_model, load_inference_model import paddle.v2.framework.executor as executor import unittest diff --git a/python/paddle/v2/framework/tests/test_layers.py b/python/paddle/v2/framework/tests/test_layers.py index 716963fb431a8..b42af5ea45d54 100644 --- a/python/paddle/v2/framework/tests/test_layers.py +++ b/python/paddle/v2/framework/tests/test_layers.py @@ -1,6 +1,6 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.nets as nets -from paddle.v2.framework.framework import Program, g_main_program +from paddle.v2.framework.framework import Program import paddle.v2.framework.core as core import unittest diff --git a/python/paddle/v2/framework/tests/test_lod_array_length_op.py b/python/paddle/v2/framework/tests/test_lod_array_length_op.py new file mode 100644 index 0000000000000..af2b4d705e7ec --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_array_length_op.py @@ -0,0 +1,21 @@ +import unittest +import paddle.v2.framework.layers as layers +from paddle.v2.framework.executor import Executor +import paddle.v2.framework.core as core +import numpy + + +class TestLoDArrayLength(unittest.TestCase): + def test_array_length(self): + tmp = layers.zeros(shape=[10], dtype='int32') + i = layers.fill_constant(shape=[1], dtype='int64', value=10) + arr = layers.array_write(tmp, i=i) + arr_len = layers.array_length(arr) + cpu = core.CPUPlace() + exe = Executor(cpu) + result = numpy.array(exe.run(fetch_list=[arr_len])[0]) + self.assertEqual(11, result[0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_reset_op.py b/python/paddle/v2/framework/tests/test_lod_reset_op.py new file mode 100644 index 0000000000000..652ccecfa443f --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_reset_op.py @@ -0,0 +1,64 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestLodResetOpByAttr(OpTest): + def setUp(self): + self.op_type = "lod_reset" + x = np.random.random((10, 20)).astype("float32") + lod = [[0, 3, 5, 10]] + target_lod_0 = [0, 7, 10] + self.inputs = {'X': (x, lod)} + self.attrs = {'target_lod': target_lod_0} + self.outputs = {'Out': (x, [target_lod_0])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out") + + +class TestLodResetOpByInput(OpTest): + def setUp(self): + self.op_type = "lod_reset" + x = np.random.random((10, 20)).astype("float32") + lod = [[0, 3, 5, 10]] + target_lod_0 = [0, 4, 7, 10] + self.inputs = { + 'X': (x, lod), + 'TargetLoD': np.array([target_lod_0]).astype('int32') + } + self.outputs = {'Out': (x, [target_lod_0])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out", no_grad_set=set("TargetLoD")) + + +class TestLodResetOpBoth(OpTest): + def setUp(self): + self.op_type = "lod_reset" + x = np.random.random((10, 20)).astype("float32") + lod = [[0, 3, 5, 10]] + target_lod_0_attr = [0, 7, 10] + target_lod_0_in = [0, 4, 7, 10] + self.inputs = { + 'X': (x, lod), + 'TargetLoD': np.array(target_lod_0_in).astype('int32') + } + self.attrs = {'target_lod': target_lod_0_attr} + self.outputs = {'Out': (x, [target_lod_0_in])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out", no_grad_set=set("TargetLoD")) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_tensor_array_ops.py b/python/paddle/v2/framework/tests/test_lod_tensor_array_ops.py index 61a5fcf07d2af..e9713666b3f64 100644 --- a/python/paddle/v2/framework/tests/test_lod_tensor_array_ops.py +++ b/python/paddle/v2/framework/tests/test_lod_tensor_array_ops.py @@ -4,6 +4,7 @@ import paddle.v2.framework.layers as layers from paddle.v2.framework.framework import Program from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops class TestCPULoDTensorArrayOps(unittest.TestCase): @@ -123,5 +124,42 @@ def check_tensor_same(self, actual, expect): self.assertEqual(actual.lod(), expect.lod()) +class TestCPULoDTensorArrayOpGrad(unittest.TestCase): + def test_grad(self): + place = core.CPUPlace() + program = Program() + + x = layers.data( + name='x', + shape=[1], + data_type='float32', + main_program=program, + stop_gradient=False) + table = layers.lod_rank_table(x, level=0, main_program=program) + array = layers.lod_tensor_to_array(x, table, main_program=program) + result = layers.array_to_lod_tensor(array, table, main_program=program) + + mean = layers.mean(x=result, main_program=program) + + append_backward_ops(mean) + + tensor = core.LoDTensor() + tensor.set(numpy.arange(10).reshape(10, 1).astype('float32'), place) + tensor.set_lod([[0, 3, 9, 10]]) + + g_vars = program.global_block().var(x.name + "@GRAD") + + exe = Executor(place) + g_out = [ + item.sum() + for item in map( + numpy.array, + exe.run(program, feed={'x': tensor}, fetch_list=[g_vars])) + ] + g_out_sum = numpy.array(g_out).sum() + + self.assertAlmostEqual(1.0, g_out_sum, delta=0.1) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lstm_op.py b/python/paddle/v2/framework/tests/test_lstm_op.py index ff75160083f29..77f062e8c8870 100644 --- a/python/paddle/v2/framework/tests/test_lstm_op.py +++ b/python/paddle/v2/framework/tests/test_lstm_op.py @@ -117,8 +117,9 @@ def set_argument(self): self.act_cell = 'tanh' self.act_cand = 'tanh' - self.has_initial_state = True + self.has_initial_state = False self.is_reverse = False + self.use_peepholes = True def setUp(self): self.set_argument() @@ -128,18 +129,28 @@ def setUp(self): N = len(self.lod[0]) - 1 x = np.random.normal(size=(T, 4 * self.D)).astype('float64') - h0 = np.zeros((N, self.D)).astype('float64') - c0 = np.zeros((N, self.D)).astype('float64') + if self.has_initial_state: + h0 = np.random.normal(size=(N, self.D)).astype('float64') + c0 = np.random.normal(size=(N, self.D)).astype('float64') + else: + h0 = np.zeros((N, self.D)).astype('float64') + c0 = np.zeros((N, self.D)).astype('float64') w = np.random.normal(size=(self.D, 4 * self.D)).astype('float64') - b = np.random.normal(size=(1, 7 * self.D)).astype('float64') + if self.use_peepholes: + b = np.random.normal(size=(1, 7 * self.D)).astype('float64') + else: + b = np.random.normal(size=(1, 4 * self.D)).astype('float64') w_b = b[:, 0:4 * self.D] - w_c = b[:, 4 * self.D:] + w_c = b[:, 4 * self.D:] if self.use_peepholes else None h, c = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse, ACTVATION[self.act_gate], ACTVATION[self.act_cell], ACTVATION[self.act_cand]) - self.inputs = {'Input': (x, self.lod), 'Weight': w, 'Bias': b} + self.inputs = {'Input': (x, self.lod), 'Weight': w} + + self.inputs['Bias'] = b + if self.has_initial_state: self.inputs['H0'] = h0 self.inputs['C0'] = c0 @@ -149,17 +160,16 @@ def setUp(self): 'Cell': (c, self.lod), } self.attrs = { - 'usePeepholes': True, - 'isReverse': self.is_reverse, - 'gateActivation': self.act_gate, - 'cellActivation': self.act_cell, - 'candidateActivation': self.act_cand + 'use_peepholes': self.use_peepholes, + 'is_reverse': self.is_reverse, + 'gate_activation': self.act_gate, + 'cell_activation': self.act_cell, + 'candidate_activation': self.act_cand } def test_check_output(self): self.check_output(atol=1e-8) - #TODO(qingqing) add more unit testing case def test_check_grad(self): # TODO(qingqing) remove folowing lines after the check_grad is refined. N = len(self.lod[0]) - 1 @@ -170,7 +180,7 @@ def test_check_grad(self): ['Input', 'Weight', 'Bias'], ['Hidden'], max_relative_error=5e-4) -class TestLstmOpHasNoInitial(TestLstmOp): +class TestLstmOpHasInitial(TestLstmOp): def set_argument(self): self.lod = [[0, 2, 5, 7]] self.D = 16 @@ -179,8 +189,69 @@ def set_argument(self): self.act_cell = 'tanh' self.act_cand = 'tanh' - self.has_initial_state = False + self.has_initial_state = True self.is_reverse = True + self.use_peepholes = True + + def test_check_grad(self): + # TODO(qingqing) remove folowing lines after the check_grad is refined. + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight', 'Bias', 'H0', 'C0'], ['Hidden'], + max_relative_error=5e-4) + + def test_check_grad_ingore_bias(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('Bias')) + + def test_check_grad_ingore_weight(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Bias'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('Weight')) + + def test_check_grad_ingore_input(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Weight', 'Bias'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('Input')) + + def test_check_grad_ingore_h0(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight', 'Bias', 'C0'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('H0')) + + def test_check_grad_ingore_c0(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight', 'Bias', 'H0'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('C0')) class TestLstmOpRerverse(TestLstmOp): @@ -192,8 +263,23 @@ def set_argument(self): self.act_cell = 'tanh' self.act_cand = 'tanh' - self.has_initial_state = True + self.has_initial_state = False + self.is_reverse = True + self.use_peepholes = True + + +class TestLstmOpNotUsePeepholes(TestLstmOp): + def set_argument(self): + self.lod = [[0, 2, 5, 7]] + self.D = 16 + + self.act_gate = 'sigmoid' + self.act_cell = 'tanh' + self.act_cand = 'tanh' + + self.has_initial_state = False self.is_reverse = True + self.use_peepholes = False if __name__ == '__main__': diff --git a/python/paddle/v2/framework/tests/test_momentum_op.py b/python/paddle/v2/framework/tests/test_momentum_op.py index 654d31975aab4..638095f7564c8 100644 --- a/python/paddle/v2/framework/tests/test_momentum_op.py +++ b/python/paddle/v2/framework/tests/test_momentum_op.py @@ -37,7 +37,7 @@ def test_check_output(self): class TestMomentumOp2(OpTest): - '''Test Momentum with defaukt values for attributes + '''Test Momentum with default values for attributes ''' def setUp(self): @@ -57,7 +57,7 @@ def setUp(self): 'LearningRate': learning_rate } - self.attrs = {'mu': mu, 'useNesterov': use_nesterov} + self.attrs = {'mu': mu, 'use_nesterov': use_nesterov} velocity_out = mu * velocity + grad if use_nesterov: diff --git a/python/paddle/v2/framework/tests/test_optimizer.py b/python/paddle/v2/framework/tests/test_optimizer.py index 9333df8f7f347..a39e7402600c7 100644 --- a/python/paddle/v2/framework/tests/test_optimizer.py +++ b/python/paddle/v2/framework/tests/test_optimizer.py @@ -98,7 +98,7 @@ def test_vanilla_momentum_optimizer(self): self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "momentum") - self.assertFalse(sgd_op.attr('useNesterov')) + self.assertFalse(sgd_op.attr('use_nesterov')) # Check accumulators accumulators = momentum_optimizer.get_accumulators() @@ -143,7 +143,7 @@ def test_nesterov_momentum_optimizer(self): self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "momentum") - self.assertTrue(sgd_op.attr('useNesterov')) + self.assertTrue(sgd_op.attr('use_nesterov')) # Check accumulators accumulators = momentum_optimizer.get_accumulators() diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index c93469e11994c..ac3fa6aa87835 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -61,8 +61,8 @@ def setUp(self): 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'poolingType': self.pool_type, - 'globalPooling': self.global_pool, + 'pooling_type': self.pool_type, + 'global_pooling': self.global_pool, } self.outputs = {'Out': output.astype('float32')} diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index 416f0df7cd27f..87483ae5e568c 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -67,8 +67,8 @@ def setUp(self): 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'poolingType': self.pool_type, - 'globalPooling': self.global_pool, + 'pooling_type': self.pool_type, + 'global_pooling': self.global_pool, } self.outputs = {'Out': output.astype('float32')} diff --git a/python/paddle/v2/framework/tests/test_pool_max_op.py b/python/paddle/v2/framework/tests/test_pool_max_op.py index cc1a867761142..04843a28ac19e 100644 --- a/python/paddle/v2/framework/tests/test_pool_max_op.py +++ b/python/paddle/v2/framework/tests/test_pool_max_op.py @@ -86,7 +86,7 @@ def setUp(self): 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'globalPooling': self.global_pool, + 'global_pooling': self.global_pool, } self.inputs = {'X': input} diff --git a/python/paddle/v2/framework/tests/test_seq_concat_op.py b/python/paddle/v2/framework/tests/test_seq_concat_op.py index abd2ebf0b21a9..dccc6ed8afe23 100644 --- a/python/paddle/v2/framework/tests/test_seq_concat_op.py +++ b/python/paddle/v2/framework/tests/test_seq_concat_op.py @@ -2,9 +2,36 @@ import numpy as np import sys from op_test import OpTest +exit(0) -class TestConcatOp(OpTest): +def to_abs_lod(lod): + if len(lod) == 0 or len(lod) == 1: + return lod + import copy + new_lod = copy.deepcopy(lod) + for idx, val in enumerate(lod[0]): + new_lod[0][idx] = lod[1][val] + return new_lod + + +def seq_concat(inputs, level): + lod0 = inputs['X'][0][1][1] + lod1 = inputs['X'][1][1][1] + x0 = inputs['X'][0][1][0] + x1 = inputs['X'][1][1][0] + level_idx = len(lod0) - level - 1 + outs = [] + for i in range(len(lod0[level_idx]) - 1): + sub_x0 = x0[to_abs_lod(lod0)[level_idx][i]:to_abs_lod(lod0)[level_idx][ + i + 1], :] + sub_x1 = x1[to_abs_lod(lod1)[level_idx][i]:to_abs_lod(lod1)[level_idx][ + i + 1], :] + outs.append(np.concatenate((sub_x0, sub_x1), axis=0)) + return np.concatenate(outs, axis=0) + + +class TestSeqConcatOp(OpTest): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 6, 3)).astype('float32') @@ -15,13 +42,7 @@ def set_data(self): level = 1 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - outs = [] - for i in range(4): - sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :] - sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :] - outs.append(np.concatenate((sub_x0, sub_x1), axis=axis)) - - self.outputs = {'Out': np.concatenate(outs, axis=0)} + self.outputs = {'Out': (np.concatenate([x0, x1], axis=1), lod0)} def setUp(self): self.op_type = "sequence_concat" @@ -34,46 +55,50 @@ def test_check_grad(self): self.check_grad(['x0'], 'Out') -class TestConcatOpDiffLod(TestConcatOp): +class TestSeqConcatOpLevelZeroNestedSequence(TestSeqConcatOp): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 6, 3)).astype('float32') lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] - x1 = np.random.random((5, 6, 3)).astype('float32') - lod1 = [[0, 3, 5], [0, 1, 2, 3, 5]] + x1 = np.random.random((7, 6, 3)).astype('float32') + lod1 = [[0, 2, 4], [0, 1, 3, 5, 7]] axis = 0 - level = 1 + level = 0 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - outs = [] - for i in range(4): - sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :] - sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :] - outs.append(np.concatenate((sub_x0, sub_x1), axis=axis)) + out_lod = [[0, 2, 4], [0, 2, 5, 8, 11]] + self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} - self.outputs = {'Out': np.concatenate(outs, axis=0)} + +class TestSeqConcatOplevelOneNestedSequence(TestSeqConcatOp): + def set_data(self): + # two level, batch size is 3 + x0 = np.random.random((4, 6, 3)).astype('float32') + lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] + x1 = np.random.random((7, 6, 3)).astype('float32') + lod1 = [[0, 3, 4], [0, 1, 3, 5, 7]] + axis = 0 + level = 1 + self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} + self.attrs = {'axis': axis, 'level': level} + out_lod = [[0, 5, 8], [0, 1, 2, 3, 5, 7, 8, 9, 11]] + self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} -class TestConcatOpLevelZero(TestConcatOp): +class TestSeqConcatOpLevelZeroSequence(TestSeqConcatOp): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 3, 4)).astype('float32') - lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] - x1 = np.random.random((5, 3, 4)).astype('float32') - lod1 = [[0, 3, 5], [0, 1, 3, 4, 5]] + lod0 = [[0, 1, 2, 3, 4]] + x1 = np.random.random((7, 3, 4)).astype('float32') + lod1 = [[0, 1, 3, 5, 7]] axis = 0 level = 0 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - outs = [] - for i in range(2): - sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :] - sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :] - outs.append(np.concatenate((sub_x0, sub_x1), axis=axis)) - - self.outputs = {'Out': np.concatenate(outs, axis=0)} + out_lod = [[0, 2, 5, 8, 11]] + self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} if __name__ == '__main__': - sys.exit(0) unittest.main() diff --git a/python/paddle/v2/framework/tests/test_shrink_rnn_memory.py b/python/paddle/v2/framework/tests/test_shrink_rnn_memory.py new file mode 100644 index 0000000000000..2090455b96980 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_shrink_rnn_memory.py @@ -0,0 +1,47 @@ +import unittest +import paddle.v2.framework.core as core +from paddle.v2.framework.executor import Executor +import paddle.v2.framework.layers as layers +from paddle.v2.framework.backward import append_backward_ops +from paddle.v2.framework.framework import g_main_program +import numpy + + +class TestShrinkRNNMemory(unittest.TestCase): + def test_shrink_rnn_memory(self): + x = layers.data('x', shape=[100], data_type='float32') + x.stop_gradient = False + table = layers.lod_rank_table(x=x) + i = layers.zeros(dtype='int64', shape=[1]) + mem1 = layers.shrink_memory(x=x, i=i, table=table) + i = layers.increment(x=i) + i.stop_gradient = True + mem2 = layers.shrink_memory(x=mem1, i=i, table=table) + i = layers.increment(x=i) + i.stop_gradient = True + mem3 = layers.shrink_memory(x=mem2, i=i, table=table) + + cpu = core.CPUPlace() + tensor = core.LoDTensor() + tensor.set_lod([[0, 2, 5, 6]]) + tensor_np = numpy.random.random(size=(3, 100)).astype('float32') + tensor.set(tensor_np, cpu) + exe = Executor(cpu) + outs = map(numpy.array, + exe.run(feed={'x': tensor}, fetch_list=[mem1, mem2, mem3])) + self.assertTrue(numpy.allclose(tensor_np[0:3], outs[0])) + self.assertTrue(numpy.allclose(tensor_np[0:2], outs[1])) + self.assertTrue(numpy.allclose(tensor_np[0:1], outs[2])) + + mem3_mean = layers.mean(x=mem3) + append_backward_ops(loss=mem3_mean) + x_grad = map(numpy.array, + exe.run(feed={'x': tensor}, + fetch_list=[ + g_main_program.global_block().var('x@GRAD') + ]))[0] + self.assertAlmostEqual(1.0, x_grad.sum(), delta=0.1) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_split_and_merge_lod_tensor_op.py b/python/paddle/v2/framework/tests/test_split_and_merge_lod_tensor_op.py new file mode 100644 index 0000000000000..6ba1e568249d4 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_split_and_merge_lod_tensor_op.py @@ -0,0 +1,181 @@ +import unittest +import paddle.v2.framework.core as core +import numpy as np +import paddle.v2.framework.layers as layers +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops + + +class TestCPULoDTensorArrayOps(unittest.TestCase): + def place(self): + return core.CPUPlace() + + def test_split_and_merge_lod_tensor_no_lod(self): + tensor = core.LoDTensor() + tensor.set(np.arange(10).reshape(10, 1).astype('int32'), self.place()) + + mask_np = np.array([0, 0, 1, 1, 1, 1, 0, 0, 0, 0]).astype('bool') + mask_np = np.expand_dims(mask_np, axis=1) + + mask = core.LoDTensor() + mask.set(mask_np, self.place()) + + expect_true_tensor = np.array([2, 3, 4, 5]).astype('int32') + expect_true_tensor = np.expand_dims(expect_true_tensor, axis=1) + expect_true = core.LoDTensor() + expect_true.set(expect_true_tensor, self.place()) + + expect_false_tensor = np.array([0, 1, 6, 7, 8, 9]).astype('int32') + expect_false_tensor = np.expand_dims(expect_false_tensor, axis=1) + + expect_false = core.LoDTensor() + expect_false.set(expect_false_tensor, self.place()) + + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor) + + def test_split_and_merge_lod_tensor_level_0(self): + tensor = core.LoDTensor() + tensor.set(np.arange(10).reshape(10, 1).astype('int32'), self.place()) + tensor.set_lod([[0, 3, 9, 10]]) + + mask_np = np.array([0, 1, 0]).astype('bool') + mask_np = np.expand_dims(mask_np, axis=1) + + mask = core.LoDTensor() + mask.set(mask_np, self.place()) + + expect_true_tensor = np.array([3, 4, 5, 6, 7, 8]).astype('int32') + expect_true_tensor = np.expand_dims(expect_true_tensor, axis=1) + expect_true = core.LoDTensor() + expect_true.set(expect_true_tensor, self.place()) + expect_true.set_lod([[0, 6]]) + + expect_false_tensor = np.array([0, 1, 2, 9]).astype('int32') + expect_false_tensor = np.expand_dims(expect_false_tensor, axis=1) + expect_false_lod = [[0, 3, 4]] + + expect_false = core.LoDTensor() + expect_false.set(expect_false_tensor, self.place()) + expect_false.set_lod(expect_false_lod) + + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor) + + def main(self, tensor, mask, expect_true, expect_false, expect_out, + level=0): + place = self.place() + program = Program() + x = layers.data(name='x', shape=[1], main_program=program) + x.persistable = True + + y = layers.data(name='y', shape=[1], main_program=program) + y.persistable = True + + out_true, out_false = layers.split_lod_tensor( + input=x, mask=y, level=level, main_program=program) + out_true.persistable = True + out_false.persistable = True + + out = layers.merge_lod_tensor( + in_true=out_true, + in_false=out_false, + mask=y, + x=x, + level=level, + main_program=program) + + out.persistable = True + + exe = Executor(place) + scope = core.Scope() + exe.run(program, feed={'x': tensor, 'y': mask}, scope=scope) + + var_true = scope.find_var(out_true.name).get_tensor() + + var_false = scope.find_var(out_false.name).get_tensor() + + var_out = scope.find_var(out.name).get_tensor() + + self.check_tensor_same(var_true, expect_true) + self.check_tensor_same(var_false, expect_false) + self.check_tensor_same(var_out, expect_out) + + def check_tensor_same(self, actual, expect): + self.assertTrue(np.allclose(np.array(actual), np.array(expect))) + self.assertEqual(actual.lod(), expect.lod()) + + +class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): + def test_grad(self): + place = core.CPUPlace() + program = Program() + + x = layers.data( + name='x', + shape=[1], + data_type='float32', + main_program=program, + stop_gradient=False) + y = layers.data( + name='y', + shape=[1], + data_type='bool', + main_program=program, + stop_gradient=False) + + level = 0 + + out_true, out_false = layers.split_lod_tensor( + input=x, mask=y, level=level, main_program=program) + out = layers.merge_lod_tensor( + in_true=out_true, + in_false=out_false, + mask=y, + x=x, + level=level, + main_program=program) + mean = layers.mean(x=out, main_program=program) + + append_backward_ops(mean) + + tensor = core.LoDTensor() + tensor.set(np.arange(10).reshape(10, 1).astype('float32'), place) + tensor.set_lod([[0, 3, 9, 10]]) + + mask_np = np.array([0, 1, 0]).astype('bool') + mask_np = np.expand_dims(mask_np, axis=1) + + mask = core.LoDTensor() + mask.set(mask_np, place) + + exe = Executor(place) + scope = core.Scope() + + g_vars = program.global_block().var(x.name + "@GRAD") + g_out = [ + item.sum() + for item in map(np.array, + exe.run(program, + feed={'x': tensor, + 'y': mask}, + fetch_list=[g_vars], + scope=scope)) + ] + + g_out_sum = np.array(g_out).sum() + + self.assertAlmostEqual(1.0, g_out_sum, delta=0.1) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_while_op.py b/python/paddle/v2/framework/tests/test_while_op.py new file mode 100644 index 0000000000000..1c344eae49705 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_while_op.py @@ -0,0 +1,68 @@ +import unittest +import paddle.v2.framework.layers as layers +from paddle.v2.framework.executor import Executor +import paddle.v2.framework.core as core +import numpy + + +class TestWhileOp(unittest.TestCase): + def test_simple_forward(self): + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, data_type='float32') + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, data_type='float32') + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, data_type='float32') + i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = True + init = layers.zeros(shape=[10], dtype='float32') + mem_array = layers.array_write(init, i=i) + data_array = layers.array_write(x=d0, i=i) + + i = layers.increment(i) + layers.array_write(d1, i, array=data_array) + + i = layers.increment(i) + layers.array_write(d2, i, array=data_array) + + i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = True + + array_len = layers.fill_constant(shape=[1], dtype='int64', value=3) + cond = layers.less_than(x=i, y=array_len) + + while_op = layers.While(cond=cond) + with while_op.block(): + d = layers.array_read(array=data_array, i=i) + prev = layers.array_read(array=mem_array, i=i) + i = layers.increment(x=i, in_place=True) + result = layers.sums(input=[d, prev]) + layers.array_write(result, i=i, array=mem_array) + layers.less_than(x=i, y=array_len, cond=cond) + sum_result = layers.array_read(mem_array, i=array_len) + + cpu = core.CPUPlace() + exe = Executor(cpu) + d = [] + + for i in xrange(3): + d.append(numpy.random.random(size=[10]).astype('float32')) + + d_tensor = [] + for item in d: + t = core.LoDTensor() + t.set(item, cpu) + d_tensor.append(t) + + outs = map(numpy.array, + exe.run(feed={ + 'd0': d_tensor[0], + 'd1': d_tensor[1], + 'd2': d_tensor[2] + }, + fetch_list=[sum_result])) + self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py index 965d965335a56..7408ea8ef611d 100644 --- a/python/paddle/v2/image.py +++ b/python/paddle/v2/image.py @@ -1,33 +1,35 @@ -import numpy as np -try: - import cv2 -except ImportError: - cv2 = None -import os -import tarfile -import cPickle - -__all__ = [ - "load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop", - "random_crop", "left_right_flip", "simple_transform", "load_and_transform", - "batch_images_from_tar" -] """ This file contains some common interfaces for image preprocess. Many users are confused about the image layout. We introduce the image layout as follows. - CHW Layout + - The abbreviations: C=channel, H=Height, W=Width - The default layout of image opened by cv2 or PIL is HWC. PaddlePaddle only supports the CHW layout. And CHW is simply a transpose of HWC. It must transpose the input image. - Color format: RGB or BGR + OpenCV use BGR color format. PIL use RGB color format. Both formats can be used for training. Noted that, the format should be keep consistent between the training and inference peroid. """ +import numpy as np +try: + import cv2 +except ImportError: + cv2 = None +import os +import tarfile +import cPickle + +__all__ = [ + "load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop", + "random_crop", "left_right_flip", "simple_transform", "load_and_transform", + "batch_images_from_tar" +] def batch_images_from_tar(data_file, @@ -36,17 +38,18 @@ def batch_images_from_tar(data_file, num_per_batch=1024): """ Read images from tar file and batch them into batch file. - param data_file: path of image tar file - type data_file: string - param dataset_name: 'train','test' or 'valid' - type dataset_name: string - param img2label: a dic with image file name as key + + :param data_file: path of image tar file + :type data_file: string + :param dataset_name: 'train','test' or 'valid' + :type dataset_name: string + :param img2label: a dic with image file name as key and image's label as value - type img2label: dic - param num_per_batch: image number per batch file - type num_per_batch: int - return: path of list file containing paths of batch file - rtype: string + :type img2label: dic + :param num_per_batch: image number per batch file + :type num_per_batch: int + :return: path of list file containing paths of batch file + :rtype: string """ batch_dir = data_file + "_batch" out_path = "%s/%s" % (batch_dir, dataset_name) @@ -99,14 +102,16 @@ def load_image_bytes(bytes, is_color=True): Example usage: .. code-block:: python + with open('cat.jpg') as f: im = load_image_bytes(f.read()) :param bytes: the input image bytes array. - :type file: str + :type bytes: str :param is_color: If set is_color True, it will load and return a color image. Otherwise, it will load and return a gray image. + :type is_color: bool """ flag = 1 if is_color else 0 file_bytes = np.asarray(bytearray(bytes), dtype=np.uint8) @@ -121,6 +126,7 @@ def load_image(file, is_color=True): Example usage: .. code-block:: python + im = load_image('cat.jpg') :param file: the input image path. @@ -128,6 +134,7 @@ def load_image(file, is_color=True): :param is_color: If set is_color True, it will load and return a color image. Otherwise, it will load and return a gray image. + :type is_color: bool """ # cv2.IMAGE_COLOR for OpenCV3 # cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version @@ -147,6 +154,7 @@ def resize_short(im, size): Example usage: .. code-block:: python + im = load_image('cat.jpg') im = resize_short(im, 256) @@ -175,6 +183,7 @@ def to_chw(im, order=(2, 0, 1)): Example usage: .. code-block:: python + im = load_image('cat.jpg') im = resize_short(im, 256) im = to_chw(im) @@ -196,6 +205,7 @@ def center_crop(im, size, is_color=True): Example usage: .. code-block:: python + im = center_crop(im, 224) :param im: the input image with HWC layout. @@ -223,6 +233,7 @@ def random_crop(im, size, is_color=True): Example usage: .. code-block:: python + im = random_crop(im, 224) :param im: the input image with HWC layout. @@ -251,6 +262,7 @@ def left_right_flip(im): Example usage: .. code-block:: python + im = left_right_flip(im) :paam im: input image with HWC layout @@ -275,6 +287,7 @@ def simple_transform(im, Example usage: .. code-block:: python + im = simple_transform(im, 256, 224, True) :param im: The input image with HWC layout. @@ -285,6 +298,11 @@ def simple_transform(im, :type crop_size: int :param is_train: Whether it is training or not. :type is_train: bool + :param is_color: whether the image is color or not. + :type is_color: bool + :param mean: the mean values, which can be element-wise mean values or + mean values per channel. + :type mean: numpy array | list """ im = resize_short(im, resize_size) if is_train: @@ -324,6 +342,7 @@ def load_and_transform(filename, Example usage: .. code-block:: python + im = load_and_transform('cat.jpg', 256, 224, True) :param filename: The file name of input image. @@ -334,6 +353,11 @@ def load_and_transform(filename, :type crop_size: int :param is_train: Whether it is training or not. :type is_train: bool + :param is_color: whether the image is color or not. + :type is_color: bool + :param mean: the mean values, which can be element-wise mean values or + mean values per channel. + :type mean: numpy array | list """ im = load_image(filename) im = simple_transform(im, resize_size, crop_size, is_train, is_color, mean) diff --git a/python/paddle/v2/optimizer.py b/python/paddle/v2/optimizer.py index 94d706b1d6289..caef5f484e2d6 100644 --- a/python/paddle/v2/optimizer.py +++ b/python/paddle/v2/optimizer.py @@ -102,7 +102,7 @@ class Momentum(Optimizer): .. math:: - v_{t} &= k * v_{t-1} - \\gamma_t / (g_{t} + \\lambda w_{t-1}) \\\\ + v_{t} &= k * v_{t-1} - \\gamma_t (g_{t} + \\lambda w_{t-1}) \\\\ w_{t} &= w_{t-1} + v_{t} \\\\ where, :math:`k` is momentum, :math:`\\lambda` is decay rate,