diff --git a/.clang-format b/.clang-format index 6bbd46d0ff956..9ba433b173624 100644 --- a/.clang-format +++ b/.clang-format @@ -13,8 +13,6 @@ # The document of clang-format is # http://clang.llvm.org/docs/ClangFormat.html # http://clang.llvm.org/docs/ClangFormatStyleOptions.html -# -# TODO(yuyang18): Add python and other language code style --- Language: Cpp BasedOnStyle: Google @@ -22,8 +20,9 @@ IndentWidth: 2 TabWidth: 2 ContinuationIndentWidth: 4 AccessModifierOffset: -2 # The private/protected/public has no indent in class -PointerAlignment: Left # int* p/int& p, not int *p/int &p Standard: Cpp11 AllowAllParametersOfDeclarationOnNextLine: true +BinPackParameters: false +BinPackArguments: false ... diff --git a/.gitignore b/.gitignore index 7e21ba0b750df..ee8489c1d71bd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,8 @@ build/ *.user .vscode -.idea \ No newline at end of file +.idea +.project +.cproject +.pydevproject +Makefile diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000..9385943da92bc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +- repo: https://github.com/Lucas-C/pre-commit-hooks.git + sha: c25201a00e6b0514370501050cf2a8538ac12270 + hooks: + - id: remove-crlf +- repo: https://github.com/reyoung/mirrors-yapf.git + sha: v0.13.2 + hooks: + - id: yapf +- repo: https://github.com/pre-commit/pre-commit-hooks + sha: 4ef03c4223ad322c7adaa6c6c0efb26b57df3b71 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + - id: end-of-file-fixer +# TODO(yuyang): trailing whitespace has some bugs on markdown +# files now, please not add it to pre-commit hook now +# - id: trailing-whitespace +# +# TODO(yuyang): debug-statements not fit for Paddle, because +# not all of our python code is runnable. Some are used for +# documenation +# - id: debug-statements diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000000..4741fb4f3bbc6 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,3 @@ +[style] +based_on_style = pep8 +column_limit = 80 diff --git a/.travis.yml b/.travis.yml index d3dae9efd416b..ffe3bc193b49e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,17 @@ language: cpp cache: ccache sudo: required dist: trusty +os: + - linux + - osx env: - JOB=DOCS - JOB=BUILD_AND_TEST +matrix: + exclude: + - os: osx + env: JOB=DOCS # Only generate documentation in linux + addons: apt: packages: @@ -27,9 +35,22 @@ addons: - libgoogle-glog-dev - libgflags-dev - libgtest-dev + - curl + - lcov + - graphviz + - swig before_install: - - pip install wheel protobuf sphinx breathe recommonmark - - sudo paddle/scripts/travis/before_install.sh + - | + if [ ${JOB} == "BUILD_AND_TEST" ]; then + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' + then + echo "Only markdown docs were updated, stopping build process." + exit + fi + fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi + - pip install wheel protobuf sphinx breathe recommonmark virtualenv numpy script: - paddle/scripts/travis/main.sh notifications: diff --git a/CMakeLists.txt b/CMakeLists.txt index af6a13efbde9e..0e6427997667a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,14 +2,14 @@ cmake_minimum_required(VERSION 2.8) project(paddle CXX C) set(PADDLE_MAJOR_VERSION 0) -set(PADDLE_MINOR_VERSION 8) -set(PADDLE_PATCH_VERSION 0b1) +set(PADDLE_MINOR_VERSION 9) +set(PADDLE_PATCH_VERSION 0) set(PADDLE_VERSION ${PADDLE_MAJOR_VERSION}.${PADDLE_MINOR_VERSION}.${PADDLE_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") set(PROJ_ROOT ${CMAKE_SOURCE_DIR}) include(package) -include(swig) +find_package(SWIG 2.0) find_package(CUDA QUIET) find_package(Protobuf REQUIRED) find_package(PythonLibs 2.7 REQUIRED) @@ -40,6 +40,9 @@ option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND}) option(WITH_DOC "Compile PaddlePaddle with documentation" OFF) option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ${SWIG_FOUND}) option(ON_TRAVIS "Running test on travis-ci or not." OFF) +option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF) +option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON) + if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel" @@ -49,11 +52,16 @@ endif() include(enableCXX11) include(cpplint) include(ccache) +if(WITH_RDMA) + include(rdma) +endif() include(util) include(flags) include(cudnn) include(FindPythonModule) include(check_packages) +include(swig) +include(coveralls) # add PaddlePaddle version if(DEFINED ENV{PADDLE_VERSION}) @@ -65,6 +73,19 @@ else() Subversion_WC_INFO(${PROJ_ROOT} Project) add_definitions(-DPADDLE_VERSION=${Project_WC_REVISION}) endif() + elseif(EXISTS ${PROJ_ROOT}/.git/) + find_package(Git REQUIRED) + execute_process( + COMMAND ${GIT_EXECUTABLE} log -1 --format=%H + WORKING_DIRECTORY ${PROJ_ROOT} + OUTPUT_VARIABLE GIT_SHA1 + RESULT_VARIABLE GIT_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT ${GIT_RESULT}) + add_definitions(-DPADDLE_VERSION=\"${GIT_SHA1}\") + else() + message(WARNING "Cannot add paddle version from git tag") + endif() endif() endif() @@ -74,11 +95,24 @@ if(NOT WITH_GPU) add_definitions(-DHPPL_STUB_FUNC) list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) else() + if(${CUDA_VERSION_MAJOR} GREATER 6) + if(COMPILER_SUPPORT_CXX11) + LIST(APPEND CUDA_NVCC_FLAGS -std=c++11) + endif() + endif() + # TODO(yuyang18): Change it to remove std=c++11 in cuda compile. set(CUDA_PROPAGATE_HOST_FLAGS OFF) if(NOT CUDNN_FOUND) message(FATAL_ERROR "Paddle need cudnn to compile") endif() + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-g -O3 --use_fast_math") + + if(WITH_AVX) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${AVX_FLAG}") + else(WITH_AVX) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SSE3_FLAG}") + endif(WITH_AVX) if(WITH_DSO) set(CUDA_LIBRARIES "") @@ -91,7 +125,7 @@ else() endif(NOT WITH_GPU) if(WITH_DOUBLE) - add_definitions(-DPADDLE_TYPE_DOUBLE -DHPPL_TYPE_DOUBLE) + add_definitions(-DPADDLE_TYPE_DOUBLE) set(ACCURACY double) else(WITH_DOUBLE) set(ACCURACY float) @@ -102,11 +136,11 @@ if(NOT WITH_TIMER) endif(NOT WITH_TIMER) if(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAG}") else(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse3") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SSE3_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SSE3_FLAG}") endif(WITH_AVX) if(WITH_PYTHON) @@ -116,12 +150,15 @@ else(WITH_PYTHON) add_definitions(-DPADDLE_NO_PYTHON) endif(WITH_PYTHON) -if(NOT WITH_RDMA) - add_definitions(-DPADDLE_DISABLE_RDMA) -endif() +if(WITH_RDMA) + include_directories("${RDMA_INC_DIR}") +else(WITH_RDMA) + add_definitions(-DPADDLE_DISABLE_RDMA) +endif(WITH_RDMA) if(WITH_GLOG) add_definitions(-DPADDLE_USE_GLOG) + include_directories(${LIBGLOG_INCLUDE_DIR}) endif() if(WITH_GFLAGS) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000000..6b2614b101108 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,14 @@ +Thank you for contributing to PaddlePaddle. Submitting an issue is a great help for us. +Both Chinese and English issues are welcome. + +It's hard to solve a problem when important details are missing. +Before submitting the issue, look over the following criteria before handing your request in. + +- [ ] Was there a similar issue submitted or resolved before ? You could search issue in the github. +- [ ] Did you retrieve your issue from widespread search engines ? +- [ ] Is my description of the issue clear enough to reproduce this problem? + * If some errors occurred, we need details about `how do you run your code?`, `what system do you use?`, `Are you using GPU or not?`, etc. + * If you use an recording [asciinema](https://asciinema.org/) to show what you are doing to make it happen, that's awesome! We could help you solve the problem more quickly. +- [ ] Is my description of the issue use the github markdown correctly? + * Please use the proper markdown syntaxes for styling all forms of writing, e.g, source code, error information, etc. + * Check out [this page](https://guides.github.com/features/mastering-markdown/) to find out much more about markdown. diff --git a/README.md b/README.md index cc2fc68ac3143..81ff8c7122ab8 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,67 @@ # PaddlePaddle -[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle) -Welcome to the PaddlePaddle GitHub. -The software will be released on Sept. 30 with full documentation and installation support. +[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle) +[![Coverage Status](https://coveralls.io/repos/github/baidu/Paddle/badge.svg?branch=develop)](https://coveralls.io/github/baidu/Paddle?branch=develop) +[![Join the chat at https://gitter.im/PaddlePaddle/Deep_Learning](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/PaddlePaddle/Deep_Learning?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE) -A pre-release version is available now for those who are eager to take a look. +Welcome to the PaddlePaddle GitHub. PaddlePaddle (PArallel Distributed Deep LEarning) is an easy-to-use, efficient, flexible and scalable deep learning platform, which is originally developed by Baidu scientists and engineers for the purpose of applying deep learning to many products at Baidu. +Our vision is to enable deep learning for everyone via PaddlePaddle. +Please refer to our [release announcement](https://github.com/baidu/Paddle/releases) to track the latest feature of PaddlePaddle. + ## Features - **Flexibility** - PaddlePaddle supports a wide range of neural network architectures and - optimization algorithms. It is easy to configure complex models such as - neural machine translation model with attention mechanism or complex memory - connection. + PaddlePaddle supports a wide range of neural network architectures and + optimization algorithms. It is easy to configure complex models such as + neural machine translation model with attention mechanism or complex memory + connection. - **Efficiency** - In order to unleash the power of heterogeneous computing resource, - optimization occurs at different levels of PaddlePaddle, including - computing, memory, architecture and communication. The following are some - examples: - 1. Optimized math operations through SSE/AVX intrinsics, BLAS libraries - (e.g. MKL, ATLAS, cuBLAS) or customized CPU/GPU kernels. - 2. Highly optimized recurrent networks which can handle **variable-length** - sequence without padding. - 3. Optimized local and distributed training for models with high dimensional - sparse data. + In order to unleash the power of heterogeneous computing resource, + optimization occurs at different levels of PaddlePaddle, including + computing, memory, architecture and communication. The following are some + examples: + + - Optimized math operations through SSE/AVX intrinsics, BLAS libraries + (e.g. MKL, ATLAS, cuBLAS) or customized CPU/GPU kernels. + - Highly optimized recurrent networks which can handle **variable-length** + sequence without padding. + - Optimized local and distributed training for models with high dimensional + sparse data. - **Scalability** - With PaddlePaddle, it is easy to use many CPUs/GPUs and machines to speed - up your training. PaddlePaddle can achieve high throughput and performance - via optimized communication. + With PaddlePaddle, it is easy to use many CPUs/GPUs and machines to speed + up your training. PaddlePaddle can achieve high throughput and performance + via optimized communication. - **Connected to Products** - In addition, PaddlePaddle is also designed to be easily deployable. At Baidu, - PaddlePaddle has been deployed into products or service with a vast number - of users, including ad click-through rate (CTR) prediction, large-scale image - classification, optical character recognition(OCR), search ranking, computer - virus detection, recommendation, etc. It is widely utilized in products at - Baidu and it has achieved a significant impact. We hope you can also exploit - the capability of PaddlePaddle to make a huge impact for your product. + In addition, PaddlePaddle is also designed to be easily deployable. At Baidu, + PaddlePaddle has been deployed into products or service with a vast number + of users, including ad click-through rate (CTR) prediction, large-scale image + classification, optical character recognition(OCR), search ranking, computer + virus detection, recommendation, etc. It is widely utilized in products at + Baidu and it has achieved a significant impact. We hope you can also exploit + the capability of PaddlePaddle to make a huge impact for your product. ## Installation -See [Installation Guide](http://paddlepaddle.org/doc/build/) to install from pre-built package or build from the source code. (Note: The installation packages are still in pre-release state and your experience of installation may not be smooth.). - +Check out the [Install Guide](http://paddlepaddle.org/doc/build/) to install from +pre-built packages (**docker image**, **deb package**) or +directly build on **Linux** and **Mac OS X** from the source code. + ## Documentation -- [Chinese Documentation](http://paddlepaddle.org/doc_cn/)
+Both [English Docs](http://paddlepaddle.org/doc/) and [Chinese Docs](http://paddlepaddle.org/doc_cn/) are provided for our users and developers. - [Quick Start](http://paddlepaddle.org/doc/demo/quick_start/index_en)
You can follow the quick start tutorial to learn how use PaddlePaddle @@ -81,9 +88,9 @@ See [Installation Guide](http://paddlepaddle.org/doc/build/) to install from pre - [Source Code Documents](http://paddlepaddle.org/doc/source/)
## Ask Questions - -If you want to ask questions and discuss about methods and models, welcome -to send email to paddle-dev@baidu.com. Framework development discussions and +Please join the [**gitter chat**](https://gitter.im/PaddlePaddle/Deep_Learning) or send email to +**paddle-dev@baidu.com** to ask questions and talk about methods and models. +Framework development discussions and bug reports are collected on [Issues](https://github.com/baidu/paddle/issues). ## Copyright and License diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000000000..a8a245ab442ba --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,69 @@ +# Release v0.9.0 + +## New Features: + +* New Layers + * bilinear interpolation layer. + * spatial pyramid-pool layer. + * de-convolution layer. + * maxout layer. +* Support rectangle padding, stride, window and input for Pooling Operation. +* Add —job=time in trainer, which can be used to print time info without compiler option -WITH_TIMER=ON. +* Expose cost_weight/nce_layer in `trainer_config_helpers` +* Add FAQ, concepts, h-rnn docs. +* Add Bidi-LSTM and DB-LSTM to quick start demo @alvations +* Add usage track scripts. + +## Improvements + +* Add Travis-CI for Mac OS X. Enable swig unittest in Travis-CI. Skip Travis-CI when only docs are changed. +* Add code coverage tools. +* Refine convolution layer to speedup and reduce GPU memory. +* Speed up PyDataProvider2 +* Add ubuntu deb package build scripts. +* Make Paddle use git-flow branching model. +* PServer support no parameter blocks. + +## Bug Fixes + +* add zlib link to py_paddle +* add input sparse data check for sparse layer at runtime +* Bug fix for sparse matrix multiplication +* Fix floating-point overflow problem of tanh +* Fix some nvcc compile options +* Fix a bug in yield dictionary in DataProvider +* Fix SRL hang when exit. + +# Release v0.8.0beta.1 +New features: + +* Mac OSX is supported by source code. #138 + * Both GPU and CPU versions of PaddlePaddle are supported. + +* Support CUDA 8.0 + +* Enhance `PyDataProvider2` + * Add dictionary yield format. `PyDataProvider2` can yield a dictionary with key is data_layer's name, value is features. + * Add `min_pool_size` to control memory pool in provider. + +* Add `deb` install package & docker image for no_avx machines. + * Especially for cloud computing and virtual machines + +* Automatically disable `avx` instructions in cmake when machine's CPU don't support `avx` instructions. + +* Add Parallel NN api in trainer_config_helpers. + +* Add `travis ci` for Github + +Bug fixes: + +* Several bugs in trainer_config_helpers. Also complete the unittest for trainer_config_helpers +* Check if PaddlePaddle is installed when unittest. +* Fix bugs in GTX series GPU +* Fix bug in MultinomialSampler + +Also more documentation was written since last release. + +# Release v0.8.0beta.0 + +PaddlePaddle v0.8.0beta.0 release. The install package is not stable yet and it's a pre-release version. diff --git a/cmake/FindAVX.cmake b/cmake/FindAVX.cmake index 58b89918ec622..d380c996dfa95 100644 --- a/cmake/FindAVX.cmake +++ b/cmake/FindAVX.cmake @@ -3,36 +3,55 @@ INCLUDE(CheckCXXSourceRuns) -SET(FIND_AVX_10) -SET(FIND_AVX_20) -SET(AVX_FLAGS) -SET(AVX_FOUND) - -# Check AVX 2 -SET(CMAKE_REQUIRED_FLAGS) IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(CMAKE_REQUIRED_FLAGS "-mavx2") -ELSEIF(MSVC AND NOT CMAKE_CL_64) # reserve for WINDOWS - SET(CMAKE_REQUIRED_FLAGS "/arch:AVX2") + set(MMX_FLAG "-mmmx") + set(SSE2_FLAG "-msse2") + set(SSE3_FLAG "-msse3") + SET(AVX_FLAG "-mavx") + SET(AVX2_FLAG "-mavx2") +ELSEIF(MSVC) + set(MMX_FLAG "/arch:MMX") + set(SSE2_FLAG "/arch:SSE2") + set(SSE3_FLAG "/arch:SSE3") + SET(AVX_FLAG "/arch:AVX") + SET(AVX2_FLAG "/arch:AVX2") ENDIF() +# Check MMX +set(CMAKE_REQUIRED_FLAGS ${MMX_FLAG}) CHECK_CXX_SOURCE_RUNS(" -#include +#include int main() { - __m256i a = _mm256_set_epi32 (-1, 2, -3, 4, -1, 2, -3, 4); - __m256i result = _mm256_abs_epi32 (a); + _mm_setzero_si64(); return 0; -}" FIND_AVX_20) +}" MMX_FOUND) -# Check AVX -SET(CMAKE_REQUIRED_FLAGS) -IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(CMAKE_REQUIRED_FLAGS "-mavx") -ELSEIF(MSVC AND NOT CMAKE_CL_64) - SET(CMAKE_REQUIRED_FLAGS "/arch:AVX") -endif() +# Check SSE2 +set(CMAKE_REQUIRED_FLAGS ${SSE2_FLAG}) +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + _mm_setzero_si128(); + return 0; +}" SSE2_FOUND) +# Check SSE3 +set(CMAKE_REQUIRED_FLAGS ${SSE3_FLAG}) +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + __m128d a = _mm_set1_pd(6.28); + __m128d b = _mm_set1_pd(3.14); + __m128d result = _mm_addsub_pd(a, b); + result = _mm_movedup_pd(result); + return 0; +}" SSE3_FOUND) + +# Check AVX +set(CMAKE_REQUIRED_FLAGS ${AVX_FLAG}) CHECK_CXX_SOURCE_RUNS(" #include int main() @@ -41,25 +60,17 @@ int main() __m256 b = _mm256_set_ps (1.0f, 2.0f, 3.0f, 4.0f, 1.0f, 2.0f, 3.0f, 4.0f); __m256 result = _mm256_add_ps (a, b); return 0; -}" FIND_AVX_10) - -IF(${FIND_AVX_20}) - IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(AVX_FLAGS "${AVX_FLAGS} -mavx2") - ELSEIF(MSVC) - SET(AVX_FLAGS "${AVX_FLAGS} /arch:AVX2") - ENDIF() -ENDIF() +}" AVX_FOUND) -IF(${FIND_AVX_10}) - IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(AVX_FLAGS "${AVX_FLAGS} -mavx") - ELSEIF(MSVC) - SET(AVX_FLAGS "${AVX_FLAGS} /arch:AVX") - ENDIF() -ENDIF() +# Check AVX 2 +set(CMAKE_REQUIRED_FLAGS ${AVX2_FLAG}) +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + __m256i a = _mm256_set_epi32 (-1, 2, -3, 4, -1, 2, -3, 4); + __m256i result = _mm256_abs_epi32 (a); + return 0; +}" AVX2_FOUND) -IF("${FIND_AVX_10}" OR "${FIND_AVX_20}") - SET(AVX_FOUND TRUE) - MESSAGE(STATUS "Find CPU supports ${AVX_FLAGS}.") -ENDIF() +mark_as_advanced(MMX_FOUND SSE2_FOUND SSE3_FOUND AVX_FOUND AVX2_FOUND) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 529b4b9d15d09..685334c658506 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -1,4 +1,4 @@ -# Find the CBlas libraries +# Find the CBlas and lapack libraries # # It will search MKL, atlas, OpenBlas, reference-cblas in order. # @@ -17,10 +17,19 @@ ## Find MKL First. set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder contains MKL") -find_path(MKL_INCLUDE_DIR mkl.h PATHS ${MKL_ROOT}/include) -find_library(MKL_CORE_LIB NAMES mkl_core PATHS ${MKL_ROOT}/lib) -find_library(MKL_SEQUENTIAL_LIB NAMES mkl_sequential PATHS ${MKL_ROOT}/lib) -find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS ${MKL_ROOT}/lib) +find_path(MKL_INCLUDE_DIR mkl.h PATHS + ${MKL_ROOT}/include) +find_path(MKL_INCLUDE_DIR mkl_lapacke.h PATHS + ${MKL_ROOT}/include) +find_library(MKL_CORE_LIB NAMES mkl_core PATHS + ${MKL_ROOT}/lib + ${MKL_ROOT}/lib/intel64) +find_library(MKL_SEQUENTIAL_LIB NAMES mkl_sequential PATHS + ${MKL_ROOT}/lib + ${MKL_ROOT}/lib/intel64) +find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS + ${MKL_ROOT}/lib + ${MKL_ROOT}/lib/intel64) if(MKL_INCLUDE_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64) @@ -30,6 +39,7 @@ if(MKL_INCLUDE_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64) ${MKL_SEQUENTIAL_LIB} ${MKL_CORE_LIB}) add_definitions(-DPADDLE_USE_MKL) + message(STATUS "Found MKL (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") return() # return file. endif() @@ -48,15 +58,19 @@ set(ATLAS_LIB_SEARCH_PATHS ) find_path(ATLAS_INC_DIR NAMES cblas.h PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) +find_path(ATLAS_CLAPACK_INC_DIR NAMES clapack.h + PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) find_library(ATLAS_CBLAS_LIB NAMES cblas libcblas.so.3 PATHS ${ATLAS_LIB_SEARCH_PATHS}) -find_library(ATLAS_LIB NAMES atlas libatlas.so.3 +find_library(ATLAS_LIB NAMES lapack_atlas liblapack_atlas.so.3 PATHS ${ATLAS_LIB_SEARCH_PATHS}) if(ATLAS_INC_DIR AND ATLAS_CBLAS_LIB AND ATLAS_LIB) set(CBLAS_PROVIDER ATLAS) - set(CBLAS_INC_DIR ${ATLAS_INC_DIR}) + set(CBLAS_INC_DIR ${ATLAS_INC_DIR} ${ATLAS_CLAPACK_INC_DIR}) set(CBLAS_LIBS ${ATLAS_LIB} ${ATLAS_CBLAS_LIB}) + add_definitions(-DPADDLE_USE_ATLAS) + message(STATUS "Found Atlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") return() endif() @@ -76,6 +90,8 @@ set(OPENBLAS_LIB_SEARCH_PATHS find_path(OPENBLAS_INC_DIR NAMES cblas.h PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS}) +find_path(OPENBLAS_LAPACKE_INC_DIR NAMES lapacke.h + PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS}) find_library(OPENBLAS_LIB NAMES openblas PATHS ${OPENBLAS_LIB_SEARCH_PATHS}) @@ -83,6 +99,7 @@ if(OPENBLAS_INC_DIR AND OPENBLAS_LIB) set(CBLAS_PROVIDER OPENBLAS) set(CBLAS_INC_DIR ${OPENBLAS_INC_DIR}) set(CBLAS_LIBS ${OPENBLAS_LIB}) + message(STATUS "Found OpenBlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") return() endif() diff --git a/cmake/coveralls.cmake b/cmake/coveralls.cmake new file mode 100644 index 0000000000000..9be7643819efd --- /dev/null +++ b/cmake/coveralls.cmake @@ -0,0 +1,103 @@ +# CMake script for code coverage. +# If _COVERALLS_UPLOAD is ON, it will upload json files to overalls.io automatically. + +# Param _COVERAGE_SRCS A list of coverage source files. +# Param _COVERALLS_UPLOAD Upload the result to coveralls. +# Param _CMAKE_SCRIPT_PATH CMake script path. +function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH) + # clean previous gcov data. + file(REMOVE_RECURSE ${PROJECT_BINARY_DIR}/*.gcda) + + # find curl for upload JSON soon. + if (_COVERALLS_UPLOAD) + find_program(CURL_EXECUTABLE curl) + if (NOT CURL_EXECUTABLE) + message(FATAL_ERROR "Coveralls: curl not found!") + endif() + endif() + + # When passing a CMake list to an external process, the list + # will be converted from the format "1;2;3" to "1 2 3". + set(COVERAGE_SRCS "") + foreach (SINGLE_SRC ${_COVERAGE_SRCS}) + set(COVERAGE_SRCS "${COVERAGE_SRCS}*${SINGLE_SRC}") + endforeach() + + # query number of logical cores + cmake_host_system_information(RESULT core_size QUERY NUMBER_OF_LOGICAL_CORES) + # coveralls json file. + set(COVERALLS_FILE ${PROJECT_BINARY_DIR}/coveralls.json) + add_custom_target(coveralls_generate + # Run regress tests. + COMMAND ${CMAKE_CTEST_COMMAND} + -j ${core_size} + --output-on-failure + # Generate Gcov and translate it into coveralls JSON. + COMMAND ${CMAKE_COMMAND} + -DCOVERAGE_SRCS="${COVERAGE_SRCS}" + -DCOVERALLS_OUTPUT_FILE="${COVERALLS_FILE}" + -DCOV_PATH="${PROJECT_BINARY_DIR}" + -DPROJECT_ROOT="${PROJECT_SOURCE_DIR}" + -P "${_CMAKE_SCRIPT_PATH}/coverallsGcovJsons.cmake" + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMENT "Coveralls: generating coveralls output..." + ) + + if (_COVERALLS_UPLOAD) + message("COVERALLS UPLOAD: ON") + # Upload the JSON to coveralls. + add_custom_target(coveralls_upload + COMMAND ${CURL_EXECUTABLE} + -S -F json_file=@${COVERALLS_FILE} + https://coveralls.io/api/v1/jobs + DEPENDS coveralls_generate + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMENT "Coveralls: uploading coveralls output...") + + add_custom_target(coveralls DEPENDS coveralls_upload) + else() + message("COVERALLS UPLOAD: OFF") + add_custom_target(coveralls DEPENDS coveralls_generate) + endif() +endfunction() + +if(ON_COVERALLS) + set(CMAKE_BUILD_TYPE "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + + set(EXCLUDE_DIRS + "demo/" + "build/" + "tests/" + ".test_env/" + ) + + if(WITH_GPU) + file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" ".c" "*.cu") + else() + file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" "*.c") + endif() + + # exclude trivial files in PADDLE_SOURCES + foreach(EXCLUDE_DIR ${EXCLUDE_DIRS}) + foreach(TMP_PATH ${PADDLE_SOURCES}) + string(FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND) + if(NOT ${EXCLUDE_DIR_FOUND} EQUAL -1) + list(REMOVE_ITEM PADDLE_SOURCES ${TMP_PATH}) + endif() + endforeach(TMP_PATH) + endforeach() + + # convert to absolute path + set(PADDLE_SRCS "") + foreach(PADDLE_SRC ${PADDLE_SOURCES}) + set(PADDLE_SRCS "${PADDLE_SRCS};${PROJECT_SOURCE_DIR}/${PADDLE_SRC}") + endforeach() + + code_coverage( + "${PADDLE_SRCS}" + ${COVERALLS_UPLOAD} + "${PROJECT_SOURCE_DIR}/cmake" + ) +endif() diff --git a/cmake/coverallsGcovJsons.cmake b/cmake/coverallsGcovJsons.cmake new file mode 100644 index 0000000000000..ae3530c3a0eeb --- /dev/null +++ b/cmake/coverallsGcovJsons.cmake @@ -0,0 +1,403 @@ +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Copyright (C) 2014 Joakim Söderberg +# +# This is intended to be run by a custom target in a CMake project like this. +# 0. Compile program with coverage support. +# 1. Clear coverage data. (Recursively delete *.gcda in build dir) +# 2. Run the unit tests. +# 3. Run this script specifying which source files the coverage should be performed on. +# +# This script will then use gcov to generate .gcov files in the directory specified +# via the COV_PATH var. This should probably be the same as your cmake build dir. +# +# It then parses the .gcov files to convert them into the Coveralls JSON format: +# https://coveralls.io/docs/api +# + +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) + +# Since it's not possible to pass a CMake list properly in the +# "1;2;3" format to an external process, we have replaced the +# ";" with "*", so reverse that here so we get it back into the +# CMake list format. +string(REGEX REPLACE "\\*" ";" COVERAGE_SRCS ${COVERAGE_SRCS}) + +find_program(GCOV_EXECUTABLE gcov) +if (NOT GCOV_EXECUTABLE) + message(FATAL_ERROR "gcov not found! Aborting...") +endif() + +find_package(Git) + +# TODO: Add these git things to the coveralls json. +if (GIT_FOUND) + # Branch. + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE GIT_BRANCH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + macro (git_log_format FORMAT_CHARS VAR_NAME) + execute_process( + COMMAND ${GIT_EXECUTABLE} log -1 --pretty=format:%${FORMAT_CHARS} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE ${VAR_NAME} + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endmacro() + + git_log_format(an GIT_AUTHOR_EMAIL) + git_log_format(ae GIT_AUTHOR_EMAIL) + git_log_format(cn GIT_COMMITTER_NAME) + git_log_format(ce GIT_COMMITTER_EMAIL) + git_log_format(B GIT_COMMIT_MESSAGE) + + message("Git exe: ${GIT_EXECUTABLE}") + message("Git branch: ${GIT_BRANCH}") + message("Git author: ${GIT_AUTHOR_NAME}") + message("Git e-mail: ${GIT_AUTHOR_EMAIL}") + message("Git commiter name: ${GIT_COMMITTER_NAME}") + message("Git commiter e-mail: ${GIT_COMMITTER_EMAIL}") + message("Git commit message: ${GIT_COMMIT_MESSAGE}") + +endif() + +############################# Macros ######################################### + +# +# This macro converts from the full path format gcov outputs: +# +# /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov +# +# to the original source file path the .gcov is for: +# +# /path/to/project/root/subdir/the_file.c +# +macro(get_source_path_from_gcov_filename _SRC_FILENAME _GCOV_FILENAME) + + # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov + # -> + # #path#to#project#root#subdir#the_file.c.gcov + get_filename_component(_GCOV_FILENAME_WEXT ${_GCOV_FILENAME} NAME) + + # #path#to#project#root#subdir#the_file.c.gcov -> /path/to/project/root/subdir/the_file.c + string(REGEX REPLACE "\\.gcov$" "" SRC_FILENAME_TMP ${_GCOV_FILENAME_WEXT}) + string(REGEX REPLACE "\#" "/" SRC_FILENAME_TMP ${SRC_FILENAME_TMP}) + set(${_SRC_FILENAME} "${SRC_FILENAME_TMP}") +endmacro() + +############################################################################## + +# Get the coverage data. +file(GLOB_RECURSE GCDA_FILES "${COV_PATH}" "*.gcda") +message("GCDA files:") + +# Get a list of all the object directories needed by gcov +# (The directories the .gcda files and .o files are found in) +# and run gcov on those. +foreach(GCDA ${GCDA_FILES}) + message("Process: ${GCDA}") + message("------------------------------------------------------------------------------") + get_filename_component(GCDA_DIR ${GCDA} PATH) + + # + # The -p below refers to "Preserve path components", + # This means that the generated gcov filename of a source file will + # keep the original files entire filepath, but / is replaced with #. + # Example: + # + # /path/to/project/root/build/CMakeFiles/the_file.dir/subdir/the_file.c.gcda + # ------------------------------------------------------------------------------ + # File '/path/to/project/root/subdir/the_file.c' + # Lines executed:68.34% of 199 + # /path/to/project/root/subdir/the_file.c:creating '#path#to#project#root#subdir#the_file.c.gcov' + # + # If -p is not specified then the file is named only "the_file.c.gcov" + # + execute_process( + COMMAND ${GCOV_EXECUTABLE} -p -o ${GCDA_DIR} ${GCDA} + WORKING_DIRECTORY ${GCDA_DIR} + ) +endforeach() + +# TODO: Make these be absolute path +file(GLOB_RECURSE ALL_GCOV_FILES "${COV_PATH}" "*.gcov") + +# Get only the filenames to use for filtering. +#set(COVERAGE_SRCS_NAMES "") +#foreach (COVSRC ${COVERAGE_SRCS}) +# get_filename_component(COVSRC_NAME ${COVSRC} NAME) +# message("${COVSRC} -> ${COVSRC_NAME}") +# list(APPEND COVERAGE_SRCS_NAMES "${COVSRC_NAME}") +#endforeach() + +# +# Filter out all but the gcov files we want. +# +# We do this by comparing the list of COVERAGE_SRCS filepaths that the +# user wants the coverage data for with the paths of the generated .gcov files, +# so that we only keep the relevant gcov files. +# +# Example: +# COVERAGE_SRCS = +# /path/to/project/root/subdir/the_file.c +# +# ALL_GCOV_FILES = +# /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov +# /path/to/project/root/build/#path#to#project#root#subdir#other_file.c.gcov +# +# Result should be: +# GCOV_FILES = +# /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov +# +set(GCOV_FILES "") +#message("Look in coverage sources: ${COVERAGE_SRCS}") +message("\nFilter out unwanted GCOV files:") +message("===============================") + +set(COVERAGE_SRCS_REMAINING ${COVERAGE_SRCS}) + +foreach (GCOV_FILE ${ALL_GCOV_FILES}) + + # + # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov + # -> + # /path/to/project/root/subdir/the_file.c + get_source_path_from_gcov_filename(GCOV_SRC_PATH ${GCOV_FILE}) + + # Is this in the list of source files? + # TODO: We want to match against relative path filenames from the source file root... + list(FIND COVERAGE_SRCS ${GCOV_SRC_PATH} WAS_FOUND) + + if (NOT WAS_FOUND EQUAL -1) + message("YES: ${GCOV_FILE}") + list(APPEND GCOV_FILES ${GCOV_FILE}) + + # We remove it from the list, so we don't bother searching for it again. + # Also files left in COVERAGE_SRCS_REMAINING after this loop ends should + # have coverage data generated from them (no lines are covered). + list(REMOVE_ITEM COVERAGE_SRCS_REMAINING ${GCOV_SRC_PATH}) + else() + message("NO: ${GCOV_FILE}") + endif() +endforeach() + +# TODO: Enable setting these +set(JSON_SERVICE_NAME "travis-ci") +set(JSON_SERVICE_JOB_ID $ENV{TRAVIS_JOB_ID}) + +set(JSON_TEMPLATE +"{ + \"service_name\": \"\@JSON_SERVICE_NAME\@\", + \"service_job_id\": \"\@JSON_SERVICE_JOB_ID\@\", + \"source_files\": \@JSON_GCOV_FILES\@ +}" +) + +set(SRC_FILE_TEMPLATE +"{ + \"name\": \"\@GCOV_SRC_REL_PATH\@\", + \"source_digest\": \"\@GCOV_CONTENTS_MD5\@\", + \"coverage\": \@GCOV_FILE_COVERAGE\@ + }" +) + +message("\nGenerate JSON for files:") +message("=========================") + +set(JSON_GCOV_FILES "[") + +# Read the GCOV files line by line and get the coverage data. +foreach (GCOV_FILE ${GCOV_FILES}) + + get_source_path_from_gcov_filename(GCOV_SRC_PATH ${GCOV_FILE}) + file(RELATIVE_PATH GCOV_SRC_REL_PATH "${PROJECT_ROOT}" "${GCOV_SRC_PATH}") + + # The new coveralls API doesn't need the entire source (Yay!) + # However, still keeping that part for now. Will cleanup in the future. + file(MD5 "${GCOV_SRC_PATH}" GCOV_CONTENTS_MD5) + message("MD5: ${GCOV_SRC_PATH} = ${GCOV_CONTENTS_MD5}") + + # Loads the gcov file as a list of lines. + # (We first open the file and replace all occurences of [] with _ + # because CMake will fail to parse a line containing unmatched brackets... + # also the \ to escaped \n in macros screws up things.) + # https://public.kitware.com/Bug/view.php?id=15369 + file(READ ${GCOV_FILE} GCOV_CONTENTS) + string(REPLACE "[" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") + string(REPLACE "]" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") + string(REPLACE "\\" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") + file(WRITE ${GCOV_FILE}_tmp "${GCOV_CONTENTS}") + + file(STRINGS ${GCOV_FILE}_tmp GCOV_LINES) + list(LENGTH GCOV_LINES LINE_COUNT) + + # Instead of trying to parse the source from the + # gcov file, simply read the file contents from the source file. + # (Parsing it from the gcov is hard because C-code uses ; in many places + # which also happens to be the same as the CMake list delimeter). + file(READ ${GCOV_SRC_PATH} GCOV_FILE_SOURCE) + + string(REPLACE "\\" "\\\\" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REGEX REPLACE "\"" "\\\\\"" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REPLACE "\t" "\\\\t" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REPLACE "\r" "\\\\r" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REPLACE "\n" "\\\\n" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + # According to http://json.org/ these should be escaped as well. + # Don't know how to do that in CMake however... + #string(REPLACE "\b" "\\\\b" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + #string(REPLACE "\f" "\\\\f" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + #string(REGEX REPLACE "\u([a-fA-F0-9]{4})" "\\\\u\\1" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + + # We want a json array of coverage data as a single string + # start building them from the contents of the .gcov + set(GCOV_FILE_COVERAGE "[") + + set(GCOV_LINE_COUNT 1) # Line number for the .gcov. + set(DO_SKIP 0) + foreach (GCOV_LINE ${GCOV_LINES}) + #message("${GCOV_LINE}") + # Example of what we're parsing: + # Hitcount |Line | Source + # " 8: 26: if (!allowed || (strlen(allowed) == 0))" + string(REGEX REPLACE + "^([^:]*):([^:]*):(.*)$" + "\\1;\\2;\\3" + RES + "${GCOV_LINE}") + + # Check if we should exclude lines using the Lcov syntax. + string(REGEX MATCH "LCOV_EXCL_START" START_SKIP "${GCOV_LINE}") + string(REGEX MATCH "LCOV_EXCL_END" END_SKIP "${GCOV_LINE}") + string(REGEX MATCH "LCOV_EXCL_LINE" LINE_SKIP "${GCOV_LINE}") + + set(RESET_SKIP 0) + if (LINE_SKIP AND NOT DO_SKIP) + set(DO_SKIP 1) + set(RESET_SKIP 1) + endif() + + if (START_SKIP) + set(DO_SKIP 1) + message("${GCOV_LINE_COUNT}: Start skip") + endif() + + if (END_SKIP) + set(DO_SKIP 0) + endif() + + list(LENGTH RES RES_COUNT) + + if (RES_COUNT GREATER 2) + list(GET RES 0 HITCOUNT) + list(GET RES 1 LINE) + list(GET RES 2 SOURCE) + + string(STRIP ${HITCOUNT} HITCOUNT) + string(STRIP ${LINE} LINE) + + # Lines with 0 line numbers are metadata and can be ignored. + if (NOT ${LINE} EQUAL 0) + + if (DO_SKIP) + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") + else() + # Translate the hitcount into valid JSON values. + if (${HITCOUNT} STREQUAL "#####") + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ") + elseif (${HITCOUNT} STREQUAL "-") + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") + else() + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}${HITCOUNT}, ") + endif() + endif() + endif() + else() + message(WARNING "Failed to properly parse line (RES_COUNT = ${RES_COUNT}) ${GCOV_FILE}:${GCOV_LINE_COUNT}\n-->${GCOV_LINE}") + endif() + + if (RESET_SKIP) + set(DO_SKIP 0) + endif() + math(EXPR GCOV_LINE_COUNT "${GCOV_LINE_COUNT}+1") + endforeach() + + message("${GCOV_LINE_COUNT} of ${LINE_COUNT} lines read!") + + # Advanced way of removing the trailing comma in the JSON array. + # "[1, 2, 3, " -> "[1, 2, 3" + string(REGEX REPLACE ",[ ]*$" "" GCOV_FILE_COVERAGE ${GCOV_FILE_COVERAGE}) + + # Append the trailing ] to complete the JSON array. + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]") + + # Generate the final JSON for this file. + message("Generate JSON for file: ${GCOV_SRC_REL_PATH}...") + string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON) + + set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ") +endforeach() + +# Loop through all files we couldn't find any coverage for +# as well, and generate JSON for those as well with 0% coverage. +foreach(NOT_COVERED_SRC ${COVERAGE_SRCS_REMAINING}) + + # Loads the source file as a list of lines. + file(STRINGS ${NOT_COVERED_SRC} SRC_LINES) + + set(GCOV_FILE_COVERAGE "[") + set(GCOV_FILE_SOURCE "") + + foreach (SOURCE ${SRC_LINES}) + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ") + + string(REPLACE "\\" "\\\\" SOURCE "${SOURCE}") + string(REGEX REPLACE "\"" "\\\\\"" SOURCE "${SOURCE}") + string(REPLACE "\t" "\\\\t" SOURCE "${SOURCE}") + string(REPLACE "\r" "\\\\r" SOURCE "${SOURCE}") + set(GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}${SOURCE}\\n") + endforeach() + + # Remove trailing comma, and complete JSON array with ] + string(REGEX REPLACE ",[ ]*$" "" GCOV_FILE_COVERAGE ${GCOV_FILE_COVERAGE}) + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]") + + # Generate the final JSON for this file. + message("Generate JSON for non-gcov file: ${NOT_COVERED_SRC}...") + string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON) + set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ") +endforeach() + +# Get rid of trailing comma. +string(REGEX REPLACE ",[ ]*$" "" JSON_GCOV_FILES ${JSON_GCOV_FILES}) +set(JSON_GCOV_FILES "${JSON_GCOV_FILES}]") + +# Generate the final complete JSON! +message("Generate final JSON...") +string(CONFIGURE ${JSON_TEMPLATE} JSON) + +file(WRITE "${COVERALLS_OUTPUT_FILE}" "${JSON}") +message("###########################################################################") +message("Generated coveralls JSON containing coverage data:") +message("${COVERALLS_OUTPUT_FILE}") +message("###########################################################################") diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 4b99e7f7fb6af..e087770991aef 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -21,12 +21,6 @@ function(safe_set_flag is_c src_list flag_name) endif() if(${safe_name}) set(${src_list} "${${src_list}} ${flag_name}" PARENT_SCOPE) - if(is_c) - set(CUDA_NVCC_FLAGS - --compiler-options;${flag_name} - ${CUDA_NVCC_FLAGS} - PARENT_SCOPE) - endif() endif() endfunction() @@ -40,6 +34,20 @@ macro(safe_set_cxxflag src_list flag_name) safe_set_flag(OFF ${src_list} ${flag_name}) endmacro() +# helper macro to set nvcc flag +macro(safe_set_nvflag flag_name) + string(REPLACE "-" "_" safe_name ${flag_name}) + string(REPLACE "=" "_" safe_name ${safe_name}) + CHECK_C_COMPILER_FLAG(${flag_name} C_COMPILER_SUPPORT_FLAG_${safe_name}) + set(safe_name C_COMPILER_SUPPORT_FLAG_${safe_name}) + if(${safe_name}) + set(CUDA_NVCC_FLAGS + --compiler-options;${flag_name} + ${CUDA_NVCC_FLAGS}) + endif() +endmacro() + + CHECK_CXX_SYMBOL_EXISTS(UINT64_MAX "stdint.h" UINT64_MAX_EXISTS) if(NOT UINT64_MAX_EXISTS) set(CMAKE_REQUIRED_DEFINITIONS -D__STDC_LIMIT_MACROS) @@ -63,14 +71,44 @@ set(COMMON_FLAGS -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter + -Wno-unused-function -Wno-error=literal-suffix -Wno-error=unused-local-typedefs) +set(GPU_COMMON_FLAGS + -fPIC + -fno-omit-frame-pointer + -Wnon-virtual-dtor + -Wdelete-non-virtual-dtor + -Wno-unused-parameter + -Wno-unused-function + -Wno-error=literal-suffix + -Wno-error=unused-local-typedefs + -Wno-error=unused-function # Warnings in Numpy Header. +) + +if (APPLE) + # On Mac OS X build fat binaries with x86_64 architectures by default. + set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE) +else() + set(GPU_COMMON_FLAGS + -Wall + -Wextra + -Werror + ${GPU_COMMON_FLAGS}) +endif() + + foreach(flag ${COMMON_FLAGS}) safe_set_cflag(CMAKE_C_FLAGS ${flag}) safe_set_cxxflag(CMAKE_CXX_FLAGS ${flag}) endforeach() +foreach(flag ${GPU_COMMON_FLAGS}) + safe_set_nvflag(${flag}) +endforeach() + + # Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc. # So, don't set these flags here. @@ -88,15 +126,15 @@ endfunction() # Common gpu architectures: Kepler, Maxwell foreach(capability 30 35 50) - list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") + list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") endforeach() -if (CUDA_VERSION VERSION_GREATER "7.0") +if (CUDA_VERSION VERSION_GREATER "7.0" OR CUDA_VERSION VERSION_EQUAL "7.0") list(APPEND __arch_flags " -gencode arch=compute_52,code=sm_52") endif() # Modern gpu architectures: Pascal -if (CUDA_VERSION VERSION_GREATER "8.0") +if (CUDA_VERSION VERSION_GREATER "8.0" OR CUDA_VERSION VERSION_EQUAL "8.0") list(APPEND __arch_flags " -gencode arch=compute_60,code=sm_60") endif() diff --git a/cmake/rdma.cmake b/cmake/rdma.cmake new file mode 100644 index 0000000000000..e9a4da79aa92a --- /dev/null +++ b/cmake/rdma.cmake @@ -0,0 +1,76 @@ +# user should download rdma first from subversion repository + +# execute following instruction to download svn mannally +# svn co https://svn.baidu.com/sys/ip/trunk/rdma/sockrdmav1 rdma/ +# svn co https://svn.baidu.com/sys/ip/trunk/rdma/thirdparty rdma/ +# we use static output in svn repositories to avoid implict bugs from not standard runtime env. + +set(RDMA_ROOT $ENV{RDMA_ROOT} CACHE PATH "Folder contains RDMA sock library and thirdparty library") + +function(generate_rdma_links) + #redirect to current DIR to isolate the pollution from system runtime environment + #it can benifits unified control for different gcc environment. + #e.g, by default gcc48 did not refer /usr/lib64 which could contain low version + #runtime libraries that will crash process while loading it. That redirect trick + #can fix it. + execute_process( + COMMAND mkdir -p librdma + COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so.1 + COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so + COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so.1 + COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) +endfunction(generate_rdma_links) + + +#check and set headers +find_path(RDMA_INC_SXISOCK sxi_sock.h PATHS ${RDMA_ROOT}/sockrdmav1/output/include) +find_path(RDMA_INC_XIO libxio.h PATHS ${RDMA_ROOT}/thirdparty/output/accelio) +find_path(RDMA_INC_EVENT event2 PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_path(RDMA_INC_NUMA numa.h PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) + +#check and set libs +find_library(RDMA_LIB_SXISOCK NAMES sxisock PATHS ${RDMA_ROOT}/sockrdmav1/output) +find_library(RDMA_LIB_XIO NAMES xio PATHS ${RDMA_ROOT}/thirdparty/output/accelio) +find_library(RDMA_LIB_EVENT NAMES event PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_EVENT_CORE NAMES event_core PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_EVENT_EXTRA NAMES event_extra PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_EVENT_PTHREADS NAMES event_pthreads PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_NUMA NAMES numa PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) + +if( + RDMA_INC_SXISOCK AND + RDMA_INC_XIO AND + RDMA_INC_EVENT AND + RDMA_INC_NUMA AND + RDMA_LIB_SXISOCK AND + RDMA_LIB_XIO AND + RDMA_LIB_EVENT AND + RDMA_LIB_EVENT_CORE AND + RDMA_LIB_EVENT_EXTRA AND + RDMA_LIB_EVENT_PTHREADS AND + RDMA_LIB_NUMA + ) + + set(RDMA_INC_DIR + ${RDMA_INC_SXISOCK} + ${RDMA_INC_XIO} + ${RDMA_INC_EVENT} + ${RDMA_INC_NUMA}) + set(RDMA_LIBS + ${RDMA_LIB_SXISOCK} + ${RDMA_LIB_XIO} + ${RDMA_LIB_EVENT} + ${RDMA_LIB_EVENT_CORE} + ${RDMA_LIB_EVENT_EXTRA} + ${RDMA_LIB_EVENT_PTHREADS} + ${RDMA_LIB_NUMA} + ) + set(RDMA_LD_FLAGS "-L./librdma -libverbs -lrdmacm -Xlinker -rpath ./librdma") + return() +endif() + +#if this module is not called, RDMA_INC_DIR RDMA_LIBS will be null, so top module always refer this variable + +message(FATAL_ERROR, "RDMA libraries are not found, try to set RDMA_ROOT or check all related libraries.") diff --git a/cmake/swig.cmake b/cmake/swig.cmake index f5c1bcc79b3dc..97e87aa947791 100644 --- a/cmake/swig.cmake +++ b/cmake/swig.cmake @@ -1,25 +1,3 @@ -find_program( - SWIG_BINARY_PATH - swig) - -if(${SWIG_BINARY_PATH} STREQUAL "SWIG_BINARY_PATH-NOTFOUND") - set(SWIG_FOUND OFF) -else() - set(SWIG_FOUND ON) -endif() - -set(MIN_SWIG_VERSION 2) -if(SWIG_FOUND) - execute_process(COMMAND sh -c "${SWIG_BINARY_PATH} -version | grep Version | cut -f3 -d' '" - OUTPUT_VARIABLE _SWIG_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(${_SWIG_VERSION} VERSION_LESS ${MIN_SWIG_VERSION}) - message("swig version ${MIN_SWIG_VERSION} or greater is needed for generating python api. " - "Only version ${_SWIG_VERSION} is found. Set SWIG_FOUND to FALSE") - set(SWIG_FOUND FALSE) - endif(${_SWIG_VERSION} VERSION_LESS ${MIN_SWIG_VERSION}) -endif(SWIG_FOUND) - function(generate_python_api target_name) add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py ${PROJ_ROOT}/paddle/Paddle_wrap.cxx @@ -27,6 +5,7 @@ function(generate_python_api target_name) COMMAND swig -python -c++ -outcurrentdir -I../ api/Paddle.swig && mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig + ${PROJ_ROOT}/paddle/api/PaddleAPI.h WORKING_DIRECTORY ${PROJ_ROOT}/paddle COMMENT "Generate Python API from swig") add_custom_target(${target_name} ALL DEPENDS diff --git a/cmake/util.cmake b/cmake/util.cmake index d776c3ae49952..a8282f07184c3 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -67,6 +67,10 @@ endmacro() # # It will handle WITH_PYTHON/WITH_GLOG etc. function(link_paddle_exe TARGET_NAME) + if(WITH_RDMA) + generate_rdma_links() + endif() + if(WITH_METRIC) if(WITH_GPU) set(METRIC_LIBS paddle_metric_learning paddle_dserver_lib metric metric_cpu) @@ -109,6 +113,12 @@ function(link_paddle_exe TARGET_NAME) ${ZLIB_LIBRARIES} ${INTERAL_LIBS} ${CMAKE_DL_LIBS}) + + if(WITH_RDMA) + target_link_libraries(${TARGET_NAME} + ${RDMA_LD_FLAGS} + ${RDMA_LIBS}) + endif() if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} @@ -178,9 +188,18 @@ macro(add_simple_unittest TARGET_NAME) add_unittest(${TARGET_NAME} ${TARGET_NAME}.cpp) endmacro() -macro(add_paddle_culib TARGET_NAME) - set(NVCC_FLAG ${CUDA_NVCC_FLAGS}) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--use_fast_math) - cuda_add_library(${TARGET_NAME} STATIC ${ARGN}) - set(CUDA_NVCC_FLAGS ${NVCC_FLAG}) -endmacro() +# Creates C resources file from files in given resource file +function(create_resources res_file output) + # Create empty output file + file(WRITE ${output} "") + # Get short filename + string(REGEX MATCH "([^/]+)$" filename ${res_file}) + # Replace filename spaces & extension separator for C compatibility + string(REGEX REPLACE "\\.| |-" "_" filename ${filename}) + # Read hex data from file + file(READ ${res_file} filedata HEX) + # Convert hex data for C compatibility + string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) + # Append data to output file + file(APPEND ${output} "const unsigned char ${filename}[] = {${filedata}};\nconst unsigned ${filename}_size = sizeof(${filename});\n") +endfunction() diff --git a/demo/image_classification/.gitignore b/demo/image_classification/.gitignore index 76961dd1436f8..6a05b8f6632db 100644 --- a/demo/image_classification/.gitignore +++ b/demo/image_classification/.gitignore @@ -5,3 +5,5 @@ plot.png train.log image_provider_copy_1.py *pyc +train.list +test.list diff --git a/demo/image_classification/data/download_cifar.sh b/demo/image_classification/data/download_cifar.sh old mode 100644 new mode 100755 diff --git a/demo/image_classification/data/process_cifar.py b/demo/image_classification/data/process_cifar.py index b766118eb0073..b235010e4ece3 100644 --- a/demo/image_classification/data/process_cifar.py +++ b/demo/image_classification/data/process_cifar.py @@ -16,7 +16,6 @@ import sys import os import PIL.Image as Image - """ Usage: python process_cifar input_dir output_dir """ @@ -30,6 +29,7 @@ def mkdir_not_exist(path): if not os.path.exists(path): os.mkdir(path) + def create_dir_structure(output_dir): """ Create the directory structure for the directory. @@ -39,8 +39,8 @@ def create_dir_structure(output_dir): mkdir_not_exist(os.path.join(output_dir, "train")) mkdir_not_exist(os.path.join(output_dir, "test")) -def convert_batch(batch_path, label_set, label_map, - output_dir, data_split): + +def convert_batch(batch_path, label_set, label_map, output_dir, data_split): """ Convert CIFAR batch to the structure of Paddle format. batch_path: the batch to be converted. @@ -67,11 +67,23 @@ def convert_batch(batch_path, label_set, label_map, output_dir = sys.argv[2] num_batch = 5 create_dir_structure(output_dir) - label_map = {0: "airplane", 1: "automobile", 2: "bird", 3: "cat", 4: "deer", - 5: "dog", 6: "frog", 7: "horse", 8: "ship", 9: "truck"} + label_map = { + 0: "airplane", + 1: "automobile", + 2: "bird", + 3: "cat", + 4: "deer", + 5: "dog", + 6: "frog", + 7: "horse", + 8: "ship", + 9: "truck" + } labels = {} for i in range(1, num_batch + 1): - convert_batch(os.path.join(input_dir, "data_batch_%d" % i), labels, - label_map, output_dir, "train") - convert_batch(os.path.join(input_dir, "test_batch"), {}, - label_map, output_dir, "test") \ No newline at end of file + convert_batch( + os.path.join(input_dir, "data_batch_%d" % i), labels, label_map, + output_dir, "train") + convert_batch( + os.path.join(input_dir, "test_batch"), {}, label_map, output_dir, + "test") diff --git a/demo/image_classification/image_provider.py b/demo/image_classification/image_provider.py index 9e2f8b8949b39..28bf1bb02c1f0 100644 --- a/demo/image_classification/image_provider.py +++ b/demo/image_classification/image_provider.py @@ -46,36 +46,41 @@ def hook(settings, img_size, mean_img_size, num_classes, color, meta, use_jpeg, settings.img_mean = image_util.load_meta(settings.meta_path, settings.mean_img_size, - settings.img_size, - settings.color) + settings.img_size, settings.color) settings.logger.info('Image size: %s', settings.img_size) settings.logger.info('Meta path: %s', settings.meta_path) settings.input_types = [ dense_vector(settings.img_raw_size), # image feature - integer_value(settings.num_classes)] # labels + integer_value(settings.num_classes) + ] # labels settings.logger.info('DataProvider Initialization finished') -@provider(init_hook=hook) -def processData(settings, file_name): +@provider(init_hook=hook, min_pool_size=0) +def processData(settings, file_list): """ The main function for loading data. Load the batch, iterate all the images and labels in this batch. - file_name: the batch file name. + file_list: the batch file list. """ - data = cPickle.load(io.open(file_name, 'rb')) - indexes = list(range(len(data['images']))) - if settings.is_train: - random.shuffle(indexes) - for i in indexes: - if settings.use_jpeg == 1: - img = image_util.decode_jpeg(data['images'][i]) - else: - img = data['images'][i] - img_feat = image_util.preprocess_img(img, settings.img_mean, - settings.img_size, settings.is_train, - settings.color) - label = data['labels'][i] - yield img_feat.tolist(), int(label) + with open(file_list, 'r') as fdata: + lines = [line.strip() for line in fdata] + random.shuffle(lines) + for file_name in lines: + with io.open(file_name.strip(), 'rb') as file: + data = cPickle.load(file) + indexes = list(range(len(data['images']))) + if settings.is_train: + random.shuffle(indexes) + for i in indexes: + if settings.use_jpeg == 1: + img = image_util.decode_jpeg(data['images'][i]) + else: + img = data['images'][i] + img_feat = image_util.preprocess_img( + img, settings.img_mean, settings.img_size, + settings.is_train, settings.color) + label = data['labels'][i] + yield img_feat.astype('float32'), int(label) diff --git a/demo/image_classification/image_util.py b/demo/image_classification/image_util.py index c545d16aafbc7..b5c6431c06f77 100644 --- a/demo/image_classification/image_util.py +++ b/demo/image_classification/image_util.py @@ -16,17 +16,20 @@ from PIL import Image from cStringIO import StringIO + def resize_image(img, target_size): """ Resize an image so that the shorter edge has length target_size. img: the input image to be resized. target_size: the target resized image size. """ - percent = (target_size/float(min(img.size[0], img.size[1]))) - resized_size = int(round(img.size[0] * percent)), int(round(img.size[1] * percent)) + percent = (target_size / float(min(img.size[0], img.size[1]))) + resized_size = int(round(img.size[0] * percent)), int( + round(img.size[1] * percent)) img = img.resize(resized_size, Image.ANTIALIAS) return img + def flip(im): """ Return the flipped image. @@ -38,6 +41,7 @@ def flip(im): else: return im[:, ::-1] + def crop_img(im, inner_size, color=True, test=True): """ Return cropped image. @@ -50,20 +54,22 @@ def crop_img(im, inner_size, color=True, test=True): If True, crop the center of images. """ if color: - height, width = max(inner_size, im.shape[1]), max(inner_size, im.shape[2]) + height, width = max(inner_size, im.shape[1]), max(inner_size, + im.shape[2]) padded_im = np.zeros((3, height, width)) startY = (height - im.shape[1]) / 2 startX = (width - im.shape[2]) / 2 endY, endX = startY + im.shape[1], startX + im.shape[2] - padded_im[:, startY: endY, startX: endX] = im + padded_im[:, startY:endY, startX:endX] = im else: im = im.astype('float32') - height, width = max(inner_size, im.shape[0]), max(inner_size, im.shape[1]) + height, width = max(inner_size, im.shape[0]), max(inner_size, + im.shape[1]) padded_im = np.zeros((height, width)) startY = (height - im.shape[0]) / 2 startX = (width - im.shape[1]) / 2 endY, endX = startY + im.shape[0], startX + im.shape[1] - padded_im[startY: endY, startX: endX] = im + padded_im[startY:endY, startX:endX] = im if test: startY = (height - inner_size) / 2 startX = (width - inner_size) / 2 @@ -72,19 +78,21 @@ def crop_img(im, inner_size, color=True, test=True): startX = np.random.randint(0, width - inner_size + 1) endY, endX = startY + inner_size, startX + inner_size if color: - pic = padded_im[:, startY: endY, startX: endX] + pic = padded_im[:, startY:endY, startX:endX] else: - pic = padded_im[startY: endY, startX: endX] + pic = padded_im[startY:endY, startX:endX] if (not test) and (np.random.randint(2) == 0): pic = flip(pic) return pic + def decode_jpeg(jpeg_string): np_array = np.array(Image.open(StringIO(jpeg_string))) if len(np_array.shape) == 3: np_array = np.transpose(np_array, (2, 0, 1)) return np_array + def preprocess_img(im, img_mean, crop_size, is_train, color=True): """ Does data augmentation for images. @@ -99,6 +107,7 @@ def preprocess_img(im, img_mean, crop_size, is_train, color=True): pic -= img_mean return pic.flatten() + def load_meta(meta_path, mean_img_size, crop_size, color=True): """ Return the loaded meta file. @@ -109,17 +118,18 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): mean = np.load(meta_path)['data_mean'] border = (mean_img_size - crop_size) / 2 if color: - assert(mean_img_size * mean_img_size * 3 == mean.shape[0]) + assert (mean_img_size * mean_img_size * 3 == mean.shape[0]) mean = mean.reshape(3, mean_img_size, mean_img_size) - mean = mean[:, border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[:, border:border + crop_size, border:border + + crop_size].astype('float32') else: - assert(mean_img_size * mean_img_size == mean.shape[0]) + assert (mean_img_size * mean_img_size == mean.shape[0]) mean = mean.reshape(mean_img_size, mean_img_size) - mean = mean[border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[border:border + crop_size, border:border + + crop_size].astype('float32') return mean + def load_image(img_path, is_color=True): """ Load image and return. @@ -130,6 +140,7 @@ def load_image(img_path, is_color=True): img.load() return img + def oversample(img, crop_dims): """ image : iterable of (H x W x K) ndarrays @@ -152,50 +163,53 @@ def oversample(img, crop_dims): for j in w_indices: crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) curr += 1 - crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate([ - -crop_dims / 2.0, - crop_dims / 2.0 - ]) + crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate( + [-crop_dims / 2.0, crop_dims / 2.0]) crops_ix = np.tile(crops_ix, (2, 1)) # Extract crops - crops = np.empty((10 * len(img), crop_dims[0], crop_dims[1], - im_shape[-1]), dtype=np.float32) + crops = np.empty( + (10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), + dtype=np.float32) ix = 0 for im in img: for crop in crops_ix: crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] ix += 1 - crops[ix-5:ix] = crops[ix-5:ix, :, ::-1, :] # flip for mirrors + crops[ix - 5:ix] = crops[ix - 5:ix, :, ::-1, :] # flip for mirrors return crops + class ImageTransformer: - def __init__(self, transpose = None, - channel_swap = None, mean = None, is_color = True): + def __init__(self, + transpose=None, + channel_swap=None, + mean=None, + is_color=True): self.transpose = transpose self.channel_swap = None self.mean = None - self.is_color = is_color + self.is_color = is_color - def set_transpose(self, order): + def set_transpose(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.transpose = order - def set_channel_swap(self, order): + def set_channel_swap(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.channel_swap = order def set_mean(self, mean): # mean value, may be one value per channel if mean.ndim == 1: - mean = mean[:, np.newaxis, np.newaxis] - else: + mean = mean[:, np.newaxis, np.newaxis] + else: # elementwise mean if self.is_color: assert len(mean.shape) == 3 - self.mean = mean + self.mean = mean def transformer(self, data): if self.transpose is not None: diff --git a/demo/image_classification/prediction.py b/demo/image_classification/prediction.py index 5d9e932658673..6a47bd5851c99 100755 --- a/demo/image_classification/prediction.py +++ b/demo/image_classification/prediction.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os,sys +import os, sys import numpy as np import logging from PIL import Image @@ -24,9 +24,11 @@ from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + class ImageClassifier(): def __init__(self, train_conf, @@ -58,18 +60,19 @@ def __init__(self, self.oversample = oversample self.is_color = is_color - self.transformer = image_util.ImageTransformer(is_color = is_color) - self.transformer.set_transpose((2,0,1)) + self.transformer = image_util.ImageTransformer(is_color=is_color) + self.transformer.set_transpose((2, 0, 1)) self.mean_file = mean_file mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) - self.transformer.set_mean(mean) # mean pixel + self.transformer.set_mean(mean) # mean pixel gpu = 1 if use_gpu else 0 conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (gpu) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (gpu)) - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) @@ -90,14 +93,14 @@ def get_data(self, img_path): # image_util.resize_image: short side is self.resize_dim image = image_util.resize_image(image, self.resize_dim) image = np.array(image) - input = np.zeros((1, image.shape[0], image.shape[1], 3), - dtype=np.float32) + input = np.zeros( + (1, image.shape[0], image.shape[1], 3), dtype=np.float32) input[0] = image.astype(np.float32) input = image_util.oversample(input, self.crop_dims) else: image = image.resize(self.crop_dims, Image.ANTIALIAS) - input = np.zeros((1, self.crop_dims[0], self.crop_dims[1], 3), - dtype=np.float32) + input = np.zeros( + (1, self.crop_dims[0], self.crop_dims[1], 3), dtype=np.float32) input[0] = np.array(image).astype(np.float32) data_in = [] @@ -133,22 +136,24 @@ def predict(self, image=None, output_layer=None): lab = np.argsort(-prob) logging.info("Label of %s is: %d", image, lab[0]) + if __name__ == '__main__': - image_size=32 - crop_size=32 - multi_crop=True - config="vgg_16_cifar.py" - output_layer="__fc_layer_1__" - mean_path="data/cifar-out/batches/batches.meta" - model_path=sys.argv[1] - image=sys.argv[2] - use_gpu=bool(int(sys.argv[3])) - - obj = ImageClassifier(train_conf=config, - model_dir=model_path, - resize_dim=image_size, - crop_dim=crop_size, - mean_file=mean_path, - use_gpu=use_gpu, - oversample=multi_crop) + image_size = 32 + crop_size = 32 + multi_crop = True + config = "vgg_16_cifar.py" + output_layer = "__fc_layer_1__" + mean_path = "data/cifar-out/batches/batches.meta" + model_path = sys.argv[1] + image = sys.argv[2] + use_gpu = bool(int(sys.argv[3])) + + obj = ImageClassifier( + train_conf=config, + model_dir=model_path, + resize_dim=image_size, + crop_dim=crop_size, + mean_file=mean_path, + use_gpu=use_gpu, + oversample=multi_crop) obj.predict(image, output_layer) diff --git a/demo/image_classification/preprocess.py b/demo/image_classification/preprocess.py index 0286a5d7e9dc8..10b9c1691b5e5 100755 --- a/demo/image_classification/preprocess.py +++ b/demo/image_classification/preprocess.py @@ -19,22 +19,36 @@ def option_parser(): parser = OptionParser(usage="usage: python preprcoess.py "\ "-i data_dir [options]") - parser.add_option("-i", "--input", action="store", - dest="input", help="Input data directory.") - parser.add_option("-s", "--size", action="store", - dest="size", help="Processed image size.") - parser.add_option("-c", "--color", action="store", - dest="color", help="whether to use color images.") + parser.add_option( + "-i", + "--input", + action="store", + dest="input", + help="Input data directory.") + parser.add_option( + "-s", + "--size", + action="store", + dest="size", + help="Processed image size.") + parser.add_option( + "-c", + "--color", + action="store", + dest="color", + help="whether to use color images.") return parser.parse_args() + if __name__ == '__main__': - options, args = option_parser() - data_dir = options.input - processed_image_size = int(options.size) - color = options.color == "1" - data_creator = ImageClassificationDatasetCreater(data_dir, - processed_image_size, - color) - data_creator.num_per_batch = 1000 - data_creator.overwrite = True - data_creator.create_batches() + options, args = option_parser() + data_dir = options.input + processed_image_size = int(options.size) + color = options.color == "1" + data_creator = ImageClassificationDatasetCreater( + data_dir, processed_image_size, color) + data_creator.train_list_name = "train.txt" + data_creator.test_list_name = "test.txt" + data_creator.num_per_batch = 1000 + data_creator.overwrite = True + data_creator.create_batches() diff --git a/demo/image_classification/preprocess.sh b/demo/image_classification/preprocess.sh index dfe3eb95d1ab8..e3e86ff10675c 100755 --- a/demo/image_classification/preprocess.sh +++ b/demo/image_classification/preprocess.sh @@ -17,3 +17,6 @@ set -e data_dir=./data/cifar-out python preprocess.py -i $data_dir -s 32 -c 1 + +echo "data/cifar-out/batches/train.txt" > train.list +echo "data/cifar-out/batches/test.txt" > test.list diff --git a/demo/image_classification/train.sh b/demo/image_classification/train.sh index ed9b5220fff6a..db0a057bf35b4 100755 --- a/demo/image_classification/train.sh +++ b/demo/image_classification/train.sh @@ -24,7 +24,7 @@ paddle train \ --test_all_data_in_one_period=1 \ --use_gpu=1 \ --trainer_count=1 \ ---num_passes=200 \ +--num_passes=300 \ --save_dir=$output \ 2>&1 | tee $log diff --git a/demo/image_classification/vgg_16_cifar.py b/demo/image_classification/vgg_16_cifar.py old mode 100644 new mode 100755 index 238608c3cbede..58ceff5fc2f46 --- a/demo/image_classification/vgg_16_cifar.py +++ b/demo/image_classification/vgg_16_cifar.py @@ -18,36 +18,38 @@ ####################Data Configuration ################## if not is_predict: - data_dir='data/cifar-out/batches/' - meta_path=data_dir+'batches.meta' - - args = {'meta':meta_path,'mean_img_size': 32, - 'img_size': 32,'num_classes': 10, - 'use_jpeg': 1,'color': "color"} - - define_py_data_sources2(train_list=data_dir+"train.list", - test_list=data_dir+'test.list', - module='image_provider', - obj='processData', - args=args) + data_dir = 'data/cifar-out/batches/' + meta_path = data_dir + 'batches.meta' + + args = { + 'meta': meta_path, + 'mean_img_size': 32, + 'img_size': 32, + 'num_classes': 10, + 'use_jpeg': 1, + 'color': "color" + } + + define_py_data_sources2( + train_list="train.list", + test_list="train.list", + module='image_provider', + obj='processData', + args=args) ######################Algorithm Configuration ############# settings( - batch_size = 128, - learning_rate = 0.1 / 128.0, - learning_method = MomentumOptimizer(0.9), - regularization = L2Regularization(0.0005 * 128) -) + batch_size=128, + learning_rate=0.1 / 128.0, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * 128)) #######################Network Configuration ############# -data_size=3*32*32 -label_size=10 -img = data_layer(name='image', - size=data_size) -# small_vgg is predined in trainer_config_helpers.network -predict = small_vgg(input_image=img, - num_channels=3, - num_classes=label_size) +data_size = 3 * 32 * 32 +label_size = 10 +img = data_layer(name='image', size=data_size) +# small_vgg is predefined in trainer_config_helpers.networks +predict = small_vgg(input_image=img, num_channels=3, num_classes=label_size) if not is_predict: lbl = data_layer(name="label", size=label_size) diff --git a/demo/introduction/README.md b/demo/introduction/README.md new file mode 100644 index 0000000000000..0614a7afe6456 --- /dev/null +++ b/demo/introduction/README.md @@ -0,0 +1,3 @@ +This folder contains scripts used in PaddlePaddle introduction. +- use `bash train.sh` to train a simple linear regression model +- use `python evaluate_model.py` to read model parameters. You can see that `w` and `b` are very close to [2, 0.3]. diff --git a/demo/introduction/dataprovider.py b/demo/introduction/dataprovider.py new file mode 100644 index 0000000000000..8515022e18dc6 --- /dev/null +++ b/demo/introduction/dataprovider.py @@ -0,0 +1,24 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.PyDataProvider2 import * +import random + + +# define data types of input: 2 real numbers +@provider(input_types=[dense_vector(1), dense_vector(1)], use_seq=False) +def process(settings, input_file): + for i in xrange(2000): + x = random.random() + yield [x], [2 * x + 0.3] diff --git a/demo/introduction/evaluate_model.py b/demo/introduction/evaluate_model.py new file mode 100755 index 0000000000000..ca4a1872731ab --- /dev/null +++ b/demo/introduction/evaluate_model.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Print model parameters in last model + +Usage: + python evaluate_model.py +""" +import numpy as np +import os + + +def load(file_name): + with open(file_name, 'rb') as f: + f.read(16) # skip header for float type. + return np.fromfile(f, dtype=np.float32) + + +def main(): + print 'w=%.6f, b=%.6f from pass 29' % (load('output/pass-00029/w'), + load('output/pass-00029/b')) + + +if __name__ == '__main__': + main() diff --git a/demo/introduction/train.sh b/demo/introduction/train.sh new file mode 100755 index 0000000000000..06db8edd105ad --- /dev/null +++ b/demo/introduction/train.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +paddle train \ + --config=trainer_config.py \ + --save_dir=./output \ + --num_passes=30 \ + 2>&1 |tee 'train.log' diff --git a/demo/introduction/trainer_config.py b/demo/introduction/trainer_config.py new file mode 100644 index 0000000000000..7c838c1a8f5b3 --- /dev/null +++ b/demo/introduction/trainer_config.py @@ -0,0 +1,41 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +# 1. read data. Suppose you saved above python code as dataprovider.py +data_file = 'empty.list' +with open(data_file, 'w') as f: + f.writelines(' ') +define_py_data_sources2( + train_list=data_file, + test_list=None, + module='dataprovider', + obj='process', + args={}) + +# 2. learning algorithm +settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) + +# 3. Network configuration +x = data_layer(name='x', size=1) +y = data_layer(name='y', size=1) +y_predict = fc_layer( + input=x, + param_attr=ParamAttr(name='w'), + size=1, + act=LinearActivation(), + bias_attr=ParamAttr(name='b')) +cost = regression_cost(input=y_predict, label=y) +outputs(cost) diff --git a/demo/mnist/.gitignore b/demo/mnist/.gitignore new file mode 100644 index 0000000000000..810910fd5ca56 --- /dev/null +++ b/demo/mnist/.gitignore @@ -0,0 +1,6 @@ +data/raw_data +data/*.list +mnist_vgg_model +plot.png +train.log +*pyc diff --git a/demo/mnist/data/generate_list.py b/demo/mnist/data/generate_list.py new file mode 100644 index 0000000000000..d880721f94c68 --- /dev/null +++ b/demo/mnist/data/generate_list.py @@ -0,0 +1,21 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +o = open("./" + "train.list", "w") +o.write("./data/raw_data/train" + "\n") +o.close() + +o = open("./" + "test.list", "w") +o.write("./data/raw_data/t10k" + "\n") +o.close() diff --git a/demo/mnist/data/get_mnist_data.sh b/demo/mnist/data/get_mnist_data.sh new file mode 100755 index 0000000000000..5a2e34026d4fe --- /dev/null +++ b/demo/mnist/data/get_mnist_data.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +# This scripts downloads the mnist data and unzips it. +set -e +DIR="$( cd "$(dirname "$0")" ; pwd -P )" +rm -rf "$DIR/raw_data" +mkdir "$DIR/raw_data" +cd "$DIR/raw_data" + +echo "Downloading..." + +for fname in train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte +do + if [ ! -e $fname ]; then + wget --no-check-certificate http://yann.lecun.com/exdb/mnist/${fname}.gz + gunzip ${fname}.gz + fi +done + +cd $DIR +rm -f *.list +python generate_list.py diff --git a/demo/mnist/mnist_provider.py b/demo/mnist/mnist_provider.py new file mode 100644 index 0000000000000..6df4676da3bdc --- /dev/null +++ b/demo/mnist/mnist_provider.py @@ -0,0 +1,31 @@ +from paddle.trainer.PyDataProvider2 import * + + +# Define a py data provider +@provider( + input_types={'pixel': dense_vector(28 * 28), + 'label': integer_value(10)}) +def process(settings, filename): # settings is not used currently. + imgf = filename + "-images-idx3-ubyte" + labelf = filename + "-labels-idx1-ubyte" + f = open(imgf, "rb") + l = open(labelf, "rb") + + f.read(16) + l.read(8) + + # Define number of samples for train/test + if "train" in filename: + n = 60000 + else: + n = 10000 + + for i in range(n): + label = ord(l.read(1)) + pixels = [] + for j in range(28 * 28): + pixels.append(float(ord(f.read(1))) / 255.0) + yield {"pixel": pixels, 'label': label} + + f.close() + l.close() diff --git a/demo/mnist/train.sh b/demo/mnist/train.sh new file mode 100755 index 0000000000000..084b32ac390b8 --- /dev/null +++ b/demo/mnist/train.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +config=vgg_16_mnist.py +output=./mnist_vgg_model +log=train.log + +paddle train \ +--config=$config \ +--dot_period=10 \ +--log_period=100 \ +--test_all_data_in_one_period=1 \ +--use_gpu=0 \ +--trainer_count=1 \ +--num_passes=100 \ +--save_dir=$output \ +2>&1 | tee $log + +python -m paddle.utils.plotcurve -i $log > plot.png diff --git a/demo/mnist/vgg_16_mnist.py b/demo/mnist/vgg_16_mnist.py new file mode 100644 index 0000000000000..f9e89bc588aba --- /dev/null +++ b/demo/mnist/vgg_16_mnist.py @@ -0,0 +1,50 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +is_predict = get_config_arg("is_predict", bool, False) + +####################Data Configuration ################## + +if not is_predict: + data_dir = './data/' + define_py_data_sources2( + train_list=data_dir + 'train.list', + test_list=data_dir + 'test.list', + module='mnist_provider', + obj='process') + +######################Algorithm Configuration ############# +settings( + batch_size=128, + learning_rate=0.1 / 128.0, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * 128)) + +#######################Network Configuration ############# + +data_size = 1 * 28 * 28 +label_size = 10 +img = data_layer(name='pixel', size=data_size) + +# small_vgg is predined in trainer_config_helpers.network +predict = small_vgg(input_image=img, num_channels=1, num_classes=label_size) + +if not is_predict: + lbl = data_layer(name="label", size=label_size) + inputs(img, lbl) + outputs(classification_cost(input=predict, label=lbl)) +else: + outputs(predict) diff --git a/demo/model_zoo/embedding/extract_para.py b/demo/model_zoo/embedding/extract_para.py index 17067792fc38d..47e06fae9caa9 100755 --- a/demo/model_zoo/embedding/extract_para.py +++ b/demo/model_zoo/embedding/extract_para.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python extract_para.py --preModel PREMODEL --preDict PREDICT \ @@ -29,6 +28,7 @@ from optparse import OptionParser import struct + def get_row_index(preDict, usrDict): """ Get the row positions for all words in user dictionary from pre-trained dictionary. @@ -47,7 +47,9 @@ def get_row_index(preDict, usrDict): pos.append(index[word]) return pos -def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, paraDim): + +def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, + paraDim): """ Extract desired parameters from a pretrained embedding model based on user dictionary """ @@ -70,6 +72,7 @@ def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, paraDim) print "extract parameters finish, total", len(rowIndex), "lines" fi.close() + def main(): """ Main entry for running paraconvert.py @@ -78,19 +81,33 @@ def main(): "python %prog --preModel PREMODEL --preDict PREDICT" \ " --usrModel USRMODEL --usrDict USRDICT -d DIM" parser = OptionParser(usage) - parser.add_option("--preModel", action="store", dest="preModel", - help="the name of pretrained embedding model") - parser.add_option("--preDict", action="store", dest="preDict", - help="the name of pretrained dictionary") - parser.add_option("--usrModel", action="store", dest="usrModel", - help="the name of output usr embedding model") - parser.add_option("--usrDict", action="store", dest="usrDict", - help="the name of user specified dictionary") - parser.add_option("-d", action="store", dest="dim", - help="dimension of parameter") + parser.add_option( + "--preModel", + action="store", + dest="preModel", + help="the name of pretrained embedding model") + parser.add_option( + "--preDict", + action="store", + dest="preDict", + help="the name of pretrained dictionary") + parser.add_option( + "--usrModel", + action="store", + dest="usrModel", + help="the name of output usr embedding model") + parser.add_option( + "--usrDict", + action="store", + dest="usrDict", + help="the name of user specified dictionary") + parser.add_option( + "-d", action="store", dest="dim", help="dimension of parameter") (options, args) = parser.parse_args() - extract_parameters_by_usrDict(options.preModel, options.preDict, - options.usrModel, options.usrDict, int(options.dim)) + extract_parameters_by_usrDict(options.preModel, options.preDict, + options.usrModel, options.usrDict, + int(options.dim)) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/embedding/paraconvert.py b/demo/model_zoo/embedding/paraconvert.py index 523412303617a..54155eff8e26b 100755 --- a/demo/model_zoo/embedding/paraconvert.py +++ b/demo/model_zoo/embedding/paraconvert.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM @@ -29,6 +28,7 @@ from optparse import OptionParser import struct + def binary2text(input, output, paraDim): """ Convert a binary parameter file of embedding model to be a text file. @@ -76,12 +76,13 @@ def binary2text(input, output, paraDim): fo.close() print "binary2text finish, total", line, "lines" + def get_para_count(input): """ Compute the total number of embedding parameters in input text file. input: the name of input text file """ - numRows = 1 + numRows = 1 paraDim = 0 with open(input) as f: line = f.readline() @@ -90,6 +91,7 @@ def get_para_count(input): numRows += 1 return numRows * paraDim + def text2binary(input, output, paddle_head=True): """ Convert a text parameter file of embedding model to be a binary file. @@ -123,6 +125,7 @@ def text2binary(input, output, paddle_head=True): fo.close() print "text2binary finish, total", count, "lines" + def main(): """ Main entry for running paraconvert.py @@ -131,21 +134,26 @@ def main(): "python %prog --b2t -i INPUT -o OUTPUT -d DIM \n" \ "python %prog --t2b -i INPUT -o OUTPUT" parser = OptionParser(usage) - parser.add_option("--b2t", action="store_true", - help="convert parameter file of embedding model from binary to text") - parser.add_option("--t2b", action="store_true", - help="convert parameter file of embedding model from text to binary") - parser.add_option("-i", action="store", dest="input", - help="input parameter file name") - parser.add_option("-o", action="store", dest="output", - help="output parameter file name") - parser.add_option("-d", action="store", dest="dim", - help="dimension of parameter") + parser.add_option( + "--b2t", + action="store_true", + help="convert parameter file of embedding model from binary to text") + parser.add_option( + "--t2b", + action="store_true", + help="convert parameter file of embedding model from text to binary") + parser.add_option( + "-i", action="store", dest="input", help="input parameter file name") + parser.add_option( + "-o", action="store", dest="output", help="output parameter file name") + parser.add_option( + "-d", action="store", dest="dim", help="dimension of parameter") (options, args) = parser.parse_args() if options.b2t: binary2text(options.input, options.output, options.dim) if options.t2b: text2binary(options.input, options.output) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/embedding/pre_DictAndModel.sh b/demo/model_zoo/embedding/pre_DictAndModel.sh index 7821850fb25cc..6d647f5dd9368 100755 --- a/demo/model_zoo/embedding/pre_DictAndModel.sh +++ b/demo/model_zoo/embedding/pre_DictAndModel.sh @@ -18,7 +18,5 @@ set -x # download the dictionary and pretrained model for file in baidu.dict model_32.emb model_64.emb model_128.emb model_256.emb do - # following is the google drive address - # you can also directly download from https://pan.baidu.com/s/1o8q577s - wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/embedding/$file --no-check-certificate + wget http://paddlepaddle.bj.bcebos.com/model_zoo/embedding/$file done diff --git a/demo/model_zoo/resnet/classify.py b/demo/model_zoo/resnet/classify.py index 06d471722f805..7855126edcfec 100755 --- a/demo/model_zoo/resnet/classify.py +++ b/demo/model_zoo/resnet/classify.py @@ -26,16 +26,22 @@ from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + class ImageClassifier(): - def __init__(self, train_conf, model_dir=None, - resize_dim=256, crop_dim=224, + def __init__(self, + train_conf, + model_dir=None, + resize_dim=256, + crop_dim=224, use_gpu=True, mean_file=None, output_layer=None, - oversample=False, is_color=True): + oversample=False, + is_color=True): """ train_conf: network configure. model_dir: string, directory of model. @@ -62,24 +68,25 @@ def __init__(self, train_conf, model_dir=None, assert isinstance(self.output_layer, basestring) self.output_layer = self.output_layer.split(",") - self.transformer = image_util.ImageTransformer(is_color = is_color) - self.transformer.set_transpose((2,0,1)) - self.transformer.set_channel_swap((2,1,0)) + self.transformer = image_util.ImageTransformer(is_color=is_color) + self.transformer.set_transpose((2, 0, 1)) + self.transformer.set_channel_swap((2, 1, 0)) self.mean_file = mean_file if self.mean_file is not None: mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) - self.transformer.set_mean(mean) # mean pixel + self.transformer.set_mean(mean) # mean pixel else: # if you use three mean value, set like: # this three mean value is calculated from ImageNet. - self.transformer.set_mean(np.array([103.939,116.779,123.68])) + self.transformer.set_mean(np.array([103.939, 116.779, 123.68])) conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (int(use_gpu)) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (int(use_gpu))) - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) @@ -105,14 +112,14 @@ def get_data(self, img_path): # image_util.resize_image: short side is self.resize_dim image = image_util.resize_image(image, self.resize_dim) image = np.array(image) - input = np.zeros((1, image.shape[0], image.shape[1], 3), - dtype=np.float32) + input = np.zeros( + (1, image.shape[0], image.shape[1], 3), dtype=np.float32) input[0] = image.astype(np.float32) input = image_util.oversample(input, self.crop_dims) else: image = image.resize(self.crop_dims, Image.ANTIALIAS) - input = np.zeros((1, self.crop_dims[0], self.crop_dims[1], 3), - dtype=np.float32) + input = np.zeros( + (1, self.crop_dims[0], self.crop_dims[1], 3), dtype=np.float32) input[0] = np.array(image).astype(np.float32) data_in = [] @@ -172,7 +179,7 @@ def predict(self, data_file): logging.info("Label of %s is: %d", image, lab[0]) return results - def extract(self, data_file, output_dir, batch_size = 10000): + def extract(self, data_file, output_dir, batch_size=10000): """ extract and save features of output layers, which are specify in Outputs() in network configure. @@ -197,7 +204,7 @@ def extract(self, data_file, output_dir, batch_size = 10000): image_feature[file_name] = feature sample_num += 1 if sample_num == batch_size: - batch_name = os.path.join(output_dir, 'batch_%d' %(batch_num)) + batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) self.save_file(image_feature, batch_name) logging.info('Finish batch %d', batch_num) batch_num += 1 @@ -206,7 +213,7 @@ def extract(self, data_file, output_dir, batch_size = 10000): if idx % 1000 == 0: logging.info('%d/%d, %s', idx, len(image_files), file_name) if sample_num > 0: - batch_name = os.path.join(output_dir, 'batch_%d' %(batch_num)) + batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) self.save_file(image_feature, batch_name) logging.info('Finish batch %d', batch_num) logging.info('Done: make image feature batch') @@ -215,38 +222,64 @@ def save_file(self, data, file): of = open(file, 'wb') cPickle.dump(data, of, protocol=cPickle.HIGHEST_PROTOCOL) + def option_parser(): """ Main entry for predciting """ usage = "%prog -c config -i data_list -w model_dir [options]" parser = OptionParser(usage="usage: %s" % usage) - parser.add_option("-j", "--job", - action="store", dest="job_type", - help="job type: predict, extract\ + parser.add_option( + "-j", + "--job", + action="store", + dest="job_type", + help="job type: predict, extract\ predict: predicting,\ extract: extract features") - parser.add_option("-c", "--conf", - action="store", dest="train_conf", - help="network config") - parser.add_option("-i", "--data", - action="store", dest="data_file", - help="image list") - parser.add_option("-w", "--model", - action="store", dest="model_path", - default=None, help="model path") - parser.add_option("-g", "--use_gpu", action="store", - dest="use_gpu", default=True, - help="Whether to use gpu mode.") - parser.add_option("-o", "--output_dir", - action="store", dest="output_dir", - default="output", help="output path") - parser.add_option("-m", "--mean", action="store", - dest="mean", default=None, - help="mean file.") - parser.add_option("-p", "--multi_crop", action="store_true", - dest="multi_crop", default=False, - help="Wether to use multiple crops on image.") + parser.add_option( + "-c", + "--conf", + action="store", + dest="train_conf", + help="network config") + parser.add_option( + "-i", "--data", action="store", dest="data_file", help="image list") + parser.add_option( + "-w", + "--model", + action="store", + dest="model_path", + default=None, + help="model path") + parser.add_option( + "-g", + "--use_gpu", + action="store", + dest="use_gpu", + default=True, + help="Whether to use gpu mode.") + parser.add_option( + "-o", + "--output_dir", + action="store", + dest="output_dir", + default="output", + help="output path") + parser.add_option( + "-m", + "--mean", + action="store", + dest="mean", + default=None, + help="mean file.") + parser.add_option( + "-p", + "--multi_crop", + action="store_true", + dest="multi_crop", + default=False, + help="Wether to use multiple crops on image.") parser.add_option("-l", "--output_layer", action="store", dest="output_layer", default=None, help="--job=extract, specify layers to extract "\ @@ -254,24 +287,26 @@ def option_parser(): "classification probability, output in resnet.py.") return parser.parse_args() + def main(): """ 1. parse input arguments. 2. predicting or extract features according job type. """ options, args = option_parser() - obj = ImageClassifier(options.train_conf, - options.model_path, - use_gpu=options.use_gpu, - mean_file=options.mean, - output_layer=options.output_layer, - oversample=options.multi_crop) + obj = ImageClassifier( + options.train_conf, + options.model_path, + use_gpu=options.use_gpu, + mean_file=options.mean, + output_layer=options.output_layer, + oversample=options.multi_crop) if options.job_type == "predict": obj.predict(options.data_file) elif options.job_type == "extract": - obj.extract(options.data_file, - options.output_dir) + obj.extract(options.data_file, options.output_dir) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/resnet/example/__init__.py b/demo/model_zoo/resnet/example/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/demo/model_zoo/resnet/example/__init__.py +++ b/demo/model_zoo/resnet/example/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/demo/model_zoo/resnet/example/image_list_provider.py b/demo/model_zoo/resnet/example/image_list_provider.py index ee457e1fffc7e..9e415f76a5332 100644 --- a/demo/model_zoo/resnet/example/image_list_provider.py +++ b/demo/model_zoo/resnet/example/image_list_provider.py @@ -16,8 +16,7 @@ from paddle.trainer.PyDataProvider2 import * -def hook(settings, image_size, crop_size, color, file_list, - is_train, **kwargs): +def hook(settings, image_size, crop_size, color, file_list, is_train, **kwargs): """ Description: Init with a list of data file file_list is the name list of input files. @@ -58,7 +57,7 @@ def hook(settings, image_size, crop_size, color, file_list, sz = settings.crop_size * settings.crop_size settings.img_mean = np.zeros(sz * 3, dtype=np.single) for idx, value in enumerate(settings.mean_value): - settings.img_mean[idx * sz: (idx + 1) * sz] = value + settings.img_mean[idx * sz:(idx + 1) * sz] = value settings.img_mean = settings.img_mean.reshape(3, settings.crop_size, settings.crop_size) @@ -69,7 +68,8 @@ def hook(settings, image_size, crop_size, color, file_list, settings.input_types = [ dense_vector(settings.img_input_size), # image feature - integer_value(1)] # labels + integer_value(1) + ] # labels settings.logger.info('Image short side: %s', settings.img_size) settings.logger.info('Crop size: %s', settings.crop_size) @@ -97,9 +97,6 @@ def processData(settings, file_list): # swap channel if settings.is_swap_channel: img = img[settings.swap_channel, :, :] - img_feat = preprocess_img(img, - settings.img_mean, - settings.crop_size, - settings.is_train, - settings.color) + img_feat = preprocess_img(img, settings.img_mean, settings.crop_size, + settings.is_train, settings.color) yield img_feat.tolist(), int(lab.strip()) diff --git a/demo/model_zoo/resnet/get_model.sh b/demo/model_zoo/resnet/get_model.sh index 89312d43edf8e..133d08fca4315 100755 --- a/demo/model_zoo/resnet/get_model.sh +++ b/demo/model_zoo/resnet/get_model.sh @@ -24,9 +24,7 @@ echo "Downloading ResNet models..." for file in resnet_50.tar.gz resnet_101.tar.gz resnet_152.tar.gz mean_meta_224.tar.gz do - # following is the google drive address - # you can also directly download from https://pan.baidu.com/s/1o8q577s - wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/imagenet/$file --no-check-certificate + wget http://paddlepaddle.bj.bcebos.com/model_zoo/imagenet/$file tar -xvf $file rm $file done diff --git a/demo/model_zoo/resnet/load_feature.py b/demo/model_zoo/resnet/load_feature.py index ee4930b7a17f7..b0948b75fd0ac 100644 --- a/demo/model_zoo/resnet/load_feature.py +++ b/demo/model_zoo/resnet/load_feature.py @@ -17,9 +17,11 @@ import cPickle import logging -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + def load_feature_c(file): """ Load feature extracted by C++ interface. @@ -30,14 +32,15 @@ def load_feature_c(file): f = open(file, 'r') for line in f: sample = [] - for slot in line.strip().split(";"): - fea = [float(val) for val in slot.strip().split()] + for slot in line.strip().split(";"): + fea = [float(val) for val in slot.strip().split()] if fea: sample.append(fea) features.append(sample) f.close() return features + def load_feature_py(feature_dir): """ Load feature extracted by python interface. @@ -54,6 +57,7 @@ def load_feature_py(feature_dir): logging.info('Load feature file %s', file_name) return features + if __name__ == '__main__': - print load_feature_py(sys.argv[1]) + print load_feature_py(sys.argv[1]) #print load_feature_c(sys.argv[1]) diff --git a/demo/model_zoo/resnet/resnet.py b/demo/model_zoo/resnet/resnet.py index 483e308ac804e..015b74cd48459 100644 --- a/demo/model_zoo/resnet/resnet.py +++ b/demo/model_zoo/resnet/resnet.py @@ -13,7 +13,6 @@ # limitations under the License. from paddle.trainer_config_helpers import * - """ paper: https://arxiv.org/abs/1512.03385 """ @@ -28,15 +27,19 @@ # mean.meta size : 3 x 224 x 224. # If you use three mean value, set like: # "mean_value:103.939,116.779,123.68;" - args={ + args = { 'mean_meta': "model/mean_meta_224/mean.meta", - 'image_size': 224, 'crop_size': 224, - 'color': True,'swap_channel:': [2, 1, 0]} - define_py_data_sources2(train_list, - 'example/test.list', - module="example.image_list_provider", - obj="processData", - args=args) + 'image_size': 224, + 'crop_size': 224, + 'color': True, + 'swap_channel:': [2, 1, 0] + } + define_py_data_sources2( + train_list, + 'example/test.list', + module="example.image_list_provider", + obj="processData", + args=args) batch_size = 1 learning_rate = 0.1 / batch_size @@ -54,12 +57,16 @@ learning_method='momentum', learning_rate_decay_a=0.5, learning_rate_decay_b=1200000 * 10, - learning_rate_schedule="discexp", -) + learning_rate_schedule="discexp", ) -def conv_bn_layer(name, input, filter_size, num_filters, - stride, padding, channels=None, +def conv_bn_layer(name, + input, + filter_size, + num_filters, + stride, + padding, + channels=None, active_type=ReluActivation()): """ A wrapper for conv layer with batch normalization layers. @@ -67,19 +74,18 @@ def conv_bn_layer(name, input, filter_size, num_filters, conv layer has no activation. """ - tmp = img_conv_layer(name=name + "_conv", - input=input, - filter_size=filter_size, - num_channels=channels, - num_filters=num_filters, - stride=stride, - padding=padding, - act=LinearActivation(), - bias_attr=False) - return batch_norm_layer(name=name + "_bn", - input=tmp, - act=active_type, - use_global_stats=is_test) + tmp = img_conv_layer( + name=name + "_conv", + input=input, + filter_size=filter_size, + num_channels=channels, + num_filters=num_filters, + stride=stride, + padding=padding, + act=LinearActivation(), + bias_attr=False) + return batch_norm_layer( + name=name + "_bn", input=tmp, act=active_type, use_global_stats=is_test) def bottleneck_block(name, input, num_filters1, num_filters2): @@ -88,29 +94,31 @@ def bottleneck_block(name, input, num_filters1, num_filters2): Last conv_bn_layer has no activation. Addto layer has activation of relu. """ - last_name = conv_bn_layer(name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=1, - padding=0) - last_name = conv_bn_layer(name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - last_name = conv_bn_layer(name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer(name=name + "_addto", - input=[input, last_name], - act=ReluActivation()) + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=1, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[input, last_name], act=ReluActivation()) def mid_projection(name, input, num_filters1, num_filters2, stride=2): @@ -123,38 +131,41 @@ def mid_projection(name, input, num_filters1, num_filters2, stride=2): branch2x: bottleneck building block, shortcuts are identity. """ # stride = 2 - branch1 = conv_bn_layer(name=name + '_branch1', - input=input, - filter_size=1, - num_filters=num_filters2, - stride=stride, - padding=0, - active_type=LinearActivation()) - - last_name = conv_bn_layer(name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=stride, - padding=0) - last_name = conv_bn_layer(name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - - last_name = conv_bn_layer(name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer(name=name + "_addto", - input=[branch1, last_name], - act=ReluActivation()) + branch1 = conv_bn_layer( + name=name + '_branch1', + input=input, + filter_size=1, + num_filters=num_filters2, + stride=stride, + padding=0, + active_type=LinearActivation()) + + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=stride, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[branch1, last_name], act=ReluActivation()) def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): @@ -168,67 +179,67 @@ def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): # For ImageNet # conv1: 112x112 img = data_layer(name='input', size=224 * 224 * 3) - tmp = conv_bn_layer("conv1", img, - filter_size=7, - channels=3, - num_filters=64, - stride=2, - padding=3) + tmp = conv_bn_layer( + "conv1", + img, + filter_size=7, + channels=3, + num_filters=64, + stride=2, + padding=3) tmp = img_pool_layer(name="pool1", input=tmp, pool_size=3, stride=2) # conv2_x: 56x56 - tmp = mid_projection(name="res2_1", - input=tmp, - num_filters1=64, - num_filters2=256, - stride=1) + tmp = mid_projection( + name="res2_1", input=tmp, num_filters1=64, num_filters2=256, stride=1) for i in xrange(2, res2_num + 1, 1): - tmp = bottleneck_block(name="res2_" + str(i), - input=tmp, - num_filters1=64, - num_filters2=256) + tmp = bottleneck_block( + name="res2_" + str(i), input=tmp, num_filters1=64, num_filters2=256) # conv3_x: 28x28 - tmp = mid_projection(name="res3_1", - input=tmp, - num_filters1=128, - num_filters2=512) + tmp = mid_projection( + name="res3_1", input=tmp, num_filters1=128, num_filters2=512) for i in xrange(2, res3_num + 1, 1): - tmp = bottleneck_block(name="res3_" + str(i), - input=tmp, num_filters1=128, - num_filters2=512) + tmp = bottleneck_block( + name="res3_" + str(i), + input=tmp, + num_filters1=128, + num_filters2=512) # conv4_x: 14x14 - tmp = mid_projection(name="res4_1", input=tmp, - num_filters1=256, num_filters2=1024) + tmp = mid_projection( + name="res4_1", input=tmp, num_filters1=256, num_filters2=1024) for i in xrange(2, res4_num + 1, 1): - tmp = bottleneck_block(name="res4_" + str(i), - input=tmp, - num_filters1=256, - num_filters2=1024) + tmp = bottleneck_block( + name="res4_" + str(i), + input=tmp, + num_filters1=256, + num_filters2=1024) # conv5_x: 7x7 - tmp = mid_projection(name="res5_1", input=tmp, - num_filters1=512, num_filters2=2048) + tmp = mid_projection( + name="res5_1", input=tmp, num_filters1=512, num_filters2=2048) for i in xrange(2, res5_num + 1, 1): - tmp = bottleneck_block(name="res5_" + str(i), - input=tmp, num_filters1=512, - num_filters2=2048) - - tmp = img_pool_layer(name='avgpool', - input=tmp, - pool_size=7, - stride=1, - pool_type=AvgPooling()) - - output = fc_layer(name='output', - input=tmp, - size=1000, - act=SoftmaxActivation()) + tmp = bottleneck_block( + name="res5_" + str(i), + input=tmp, + num_filters1=512, + num_filters2=2048) + + tmp = img_pool_layer( + name='avgpool', + input=tmp, + pool_size=7, + stride=1, + pool_type=AvgPooling()) + + output = fc_layer( + name='output', input=tmp, size=1000, act=SoftmaxActivation()) if not is_predict: - classification_cost(input=output, label=data_layer(name='label', - size=1)) + classification_cost( + input=output, label=data_layer( + name='label', size=1)) def res_net_50(): diff --git a/demo/quick_start/api_train.py b/demo/quick_start/api_train.py new file mode 100644 index 0000000000000..66cbb856484d2 --- /dev/null +++ b/demo/quick_start/api_train.py @@ -0,0 +1,122 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import itertools +import random + +from paddle.trainer.config_parser import parse_config +from py_paddle import swig_paddle as api +from py_paddle import DataProviderConverter +from paddle.trainer.PyDataProvider2 \ + import integer_value, integer_value_sequence, sparse_binary_vector + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--train_data", type=str, required=False, help="train data file") + parser.add_argument("--test_data", type=str, help="test data file") + parser.add_argument( + "--config", type=str, required=True, help="config file name") + parser.add_argument("--dict_file", required=True, help="dictionary file") + parser.add_argument( + "--seq", default=1, type=int, help="whether use sequence training") + parser.add_argument( + "--use_gpu", default=0, type=int, help="whether use GPU for training") + parser.add_argument( + "--trainer_count", + default=1, + type=int, + help="Number of threads for training") + parser.add_argument( + "--num_passes", default=5, type=int, help="Number of training passes") + return parser.parse_args() + + +UNK_IDX = 0 + + +def load_data(file_name, word_dict): + with open(file_name, 'r') as f: + for line in f: + label, comment = line.strip().split('\t') + words = comment.split() + word_slot = [word_dict.get(w, UNK_IDX) for w in words] + yield word_slot, int(label) + + +def load_dict(dict_file): + word_dict = dict() + with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + return word_dict + + +def main(): + options = parse_arguments() + api.initPaddle("--use_gpu=%s" % options.use_gpu, + "--trainer_count=%s" % options.trainer_count) + + word_dict = load_dict(options.dict_file) + train_dataset = list(load_data(options.train_data, word_dict)) + if options.test_data: + test_dataset = list(load_data(options.test_data, word_dict)) + else: + test_dataset = None + + trainer_config = parse_config(options.config, + "dict_file=%s" % options.dict_file) + # No need to have data provider for trainer + trainer_config.ClearField('data_config') + trainer_config.ClearField('test_data_config') + + # create a GradientMachine from the model configuratin + model = api.GradientMachine.createFromConfigProto( + trainer_config.model_config) + # create a trainer for the gradient machine + trainer = api.Trainer.create(trainer_config, model) + + # create a data converter which converts data to PaddlePaddle + # internal format + input_types = [ + integer_value_sequence(len(word_dict)) if options.seq else + sparse_binary_vector(len(word_dict)), integer_value(2) + ] + converter = DataProviderConverter(input_types) + + batch_size = trainer_config.opt_config.batch_size + trainer.startTrain() + for train_pass in xrange(options.num_passes): + trainer.startTrainPass() + random.shuffle(train_dataset) + for pos in xrange(0, len(train_dataset), batch_size): + batch = itertools.islice(train_dataset, pos, pos + batch_size) + size = min(batch_size, len(train_dataset) - pos) + trainer.trainOneDataBatch(size, converter(batch)) + trainer.finishTrainPass() + if test_dataset: + trainer.startTestPeriod() + for pos in xrange(0, len(test_dataset), batch_size): + batch = itertools.islice(test_dataset, pos, pos + batch_size) + size = min(batch_size, len(test_dataset) - pos) + trainer.testOneDataBatch(size, converter(batch)) + trainer.finishTestPeriod() + trainer.finishTrain() + + +if __name__ == '__main__': + main() diff --git a/demo/quick_start/api_train.sh b/demo/quick_start/api_train.sh new file mode 100755 index 0000000000000..40e9d0a09aaa6 --- /dev/null +++ b/demo/quick_start/api_train.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +# Note: if using trainer_config.emb.py, trainer_config.cnn.py +# or trainer_config.lstm.py, you need to change --seq to --seq=1 +# because they are sequence models. +python api_train.py \ + --config=trainer_config.lr.py \ + --trainer_count=2 \ + --num_passes=15 \ + --use_gpu=0 \ + --seq=0 \ + --train_data=data/train.txt \ + --test_data=data/test.txt \ + --dict_file=data/dict.txt \ + 2>&1 | tee 'train.log' diff --git a/demo/quick_start/dataprovider_bow.py b/demo/quick_start/dataprovider_bow.py index 435e6d8175bd6..a5156a2d40cc0 100644 --- a/demo/quick_start/dataprovider_bow.py +++ b/demo/quick_start/dataprovider_bow.py @@ -17,6 +17,7 @@ # id of the word not in dictionary UNK_IDX = 0 + # initializer is called by the framework during initialization. # It allows the user to describe the data types and setup the # necessary data structure for later use. @@ -38,7 +39,9 @@ def initializer(settings, dictionary, **kwargs): # The second input is an integer. It represents the category id of the # sample. 2 means there are two labels in the dataset. # (1 for positive and 0 for negative) - integer_value(2)] + integer_value(2) + ] + # Delaring a data provider. It has an initializer 'data_initialzer'. # It will cache the generated data of the first pass in memory, so that @@ -69,9 +72,8 @@ def process(settings, file_name): def predict_initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary - settings.input_types = [ - sparse_binary_vector(len(dictionary)) - ] + settings.input_types = [sparse_binary_vector(len(dictionary))] + # Declaring a data provider for prediction. The difference with process # is that label is not generated. @@ -79,6 +81,6 @@ def predict_initializer(settings, dictionary, **kwargs): def process_predict(settings, file_name): with open(file_name, 'r') as f: for line in f: - comment = line.strip() + comment = line.strip().split() word_vector = [settings.word_dict.get(w, UNK_IDX) for w in comment] yield word_vector diff --git a/demo/quick_start/dataprovider_emb.py b/demo/quick_start/dataprovider_emb.py index e5030c5e71aa5..286f3f5c82081 100755 --- a/demo/quick_start/dataprovider_emb.py +++ b/demo/quick_start/dataprovider_emb.py @@ -16,6 +16,7 @@ UNK_IDX = 0 + def initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ @@ -23,7 +24,8 @@ def initializer(settings, dictionary, **kwargs): # The value of the integers range from 0 to len(dictrionary)-1 integer_value_sequence(len(dictionary)), # Define the second input for label id - integer_value(2)] + integer_value(2) + ] @provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) @@ -39,7 +41,8 @@ def process(settings, file_name): def predict_initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ - integer_value(len(dictionary), seq_type=SequenceType.SEQUENCE) + integer_value( + len(dictionary), seq_type=SequenceType.SEQUENCE) ] @@ -47,6 +50,6 @@ def predict_initializer(settings, dictionary, **kwargs): def process_predict(settings, file_name): with open(file_name, 'r') as f: for line in f: - comment = line.strip() + comment = line.strip().split() word_slot = [settings.word_dict.get(w, UNK_IDX) for w in comment] yield word_slot diff --git a/demo/quick_start/preprocess.py b/demo/quick_start/preprocess.py index 69fdbe44b5245..d87fad632a742 100755 --- a/demo/quick_start/preprocess.py +++ b/demo/quick_start/preprocess.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ 1. (remove HTML before or not)tokensizing 2. pos sample : rating score 5; neg sample: rating score 1-2. @@ -35,7 +34,8 @@ batch_size = 5000 word_count = {} -num_tokenize = max(1, multiprocessing.cpu_count() - 2) # parse + tokenize + save +num_tokenize = max(1, + multiprocessing.cpu_count() - 2) # parse + tokenize + save max_queue_size = 8 parse_queue = Queue(maxsize=max_queue_size + num_tokenize) tokenize_queue = Queue(maxsize=max_queue_size + num_tokenize) diff --git a/demo/quick_start/preprocess.sh b/demo/quick_start/preprocess.sh index fb2bee98beb26..58a72147c5e41 100755 --- a/demo/quick_start/preprocess.sh +++ b/demo/quick_start/preprocess.sh @@ -20,13 +20,22 @@ set -e +export LC_ALL=C +UNAME_STR=`uname` + +if [[ ${UNAME_STR} == 'Linux' ]]; then + SHUF_PROG='shuf' +else + SHUF_PROG='gshuf' +fi + mkdir -p data/tmp python preprocess.py -i data/reviews_Electronics_5.json.gz # uniq and shuffle cd data/tmp echo 'uniq and shuffle...' -cat pos_*|sort|uniq|shuf> pos.shuffed -cat neg_*|sort|uniq|shuf> neg.shuffed +cat pos_*|sort|uniq|${SHUF_PROG}> pos.shuffed +cat neg_*|sort|uniq|${SHUF_PROG}> neg.shuffed min_len=`sed -n '$=' neg.shuffed` test_num=$((min_len/10)) @@ -40,8 +49,8 @@ head -n$train_num neg.shuffed >train.neg tail -n$test_num pos.shuffed >test.pos tail -n$test_num neg.shuffed >test.neg -cat train.pos train.neg|shuf>../train.txt -cat test.pos test.neg|shuf>../test.txt +cat train.pos train.neg | ${SHUF_PROG} >../train.txt +cat test.pos test.neg | ${SHUF_PROG} >../test.txt cd - echo 'data/train.txt' > data/train.list diff --git a/demo/quick_start/train.sh b/demo/quick_start/train.sh index 1f0a137c8bd59..b3c471608c324 100755 --- a/demo/quick_start/train.sh +++ b/demo/quick_start/train.sh @@ -18,11 +18,14 @@ cfg=trainer_config.lr.py #cfg=trainer_config.emb.py #cfg=trainer_config.cnn.py #cfg=trainer_config.lstm.py +#cfg=trainer_config.bidi-lstm.py +#cfg=trainer_config.db-lstm.py +#cfg=trainer_config.resnet-lstm.py paddle train \ --config=$cfg \ --save_dir=./output \ --trainer_count=4 \ - --log_period=20 \ + --log_period=100 \ --num_passes=15 \ --use_gpu=false \ --show_parameter_stats_period=100 \ diff --git a/demo/quick_start/trainer_config.bidi-lstm.py b/demo/quick_start/trainer_config.bidi-lstm.py new file mode 100644 index 0000000000000..51deaf31f9468 --- /dev/null +++ b/demo/quick_start/trainer_config.bidi-lstm.py @@ -0,0 +1,61 @@ +# edit-mode: -*- python -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +dict_file = "./data/dict.txt" +word_dict = dict() +with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + +is_predict = get_config_arg('is_predict', bool, False) +trn = 'data/train.list' if not is_predict else None +tst = 'data/test.list' if not is_predict else 'data/pred.list' +process = 'process' if not is_predict else 'process_predict' +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) + +batch_size = 128 if not is_predict else 1 +settings( + batch_size=batch_size, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25) + +bias_attr = ParamAttr(initial_std=0., l2_rate=0.) +data = data_layer(name="word", size=len(word_dict)) +emb = embedding_layer(input=data, size=128) + +bi_lstm = bidirectional_lstm(input=emb, size=128) +dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) + +output = fc_layer( + input=dropout, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) + +if is_predict: + maxid = maxid_layer(output) + outputs([maxid, output]) +else: + label = data_layer(name="label", size=2) + cls = classification_cost(input=output, label=label) + outputs(cls) diff --git a/demo/quick_start/trainer_config.cnn.py b/demo/quick_start/trainer_config.cnn.py index 253ec0aee26cf..388efa75f903e 100644 --- a/demo/quick_start/trainer_config.cnn.py +++ b/demo/quick_start/trainer_config.cnn.py @@ -27,11 +27,12 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,8 +40,7 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) data = data_layer(name="word", size=len(word_dict)) embedding = embedding_layer(input=data, size=128) diff --git a/demo/quick_start/trainer_config.db-lstm.py b/demo/quick_start/trainer_config.db-lstm.py new file mode 100644 index 0000000000000..02bc898d881ef --- /dev/null +++ b/demo/quick_start/trainer_config.db-lstm.py @@ -0,0 +1,74 @@ +# edit-mode: -*- python -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +dict_file = "./data/dict.txt" +word_dict = dict() +with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + +is_predict = get_config_arg('is_predict', bool, False) +trn = 'data/train.list' if not is_predict else None +tst = 'data/test.list' if not is_predict else 'data/pred.list' +process = 'process' if not is_predict else 'process_predict' +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) + +batch_size = 128 if not is_predict else 1 +settings( + batch_size=batch_size, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25) + +bias_attr = ParamAttr(initial_std=0., l2_rate=0.) + +data = data_layer(name="word", size=len(word_dict)) +emb = embedding_layer(input=data, size=128) + +hidden_0 = mixed_layer(size=128, input=[full_matrix_projection(input=emb)]) +lstm_0 = lstmemory(input=hidden_0, layer_attr=ExtraAttr(drop_rate=0.1)) + +input_layers = [hidden_0, lstm_0] + +for i in range(1, 8): + fc = fc_layer(input=input_layers, size=128) + lstm = lstmemory( + input=fc, + layer_attr=ExtraAttr(drop_rate=0.1), + reverse=(i % 2) == 1, ) + input_layers = [fc, lstm] + +lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) + +output = fc_layer( + input=lstm_last, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) + +if is_predict: + maxid = maxid_layer(output) + outputs([maxid, output]) +else: + label = data_layer(name="label", size=2) + cls = classification_cost(input=output, label=label) + outputs(cls) diff --git a/demo/quick_start/trainer_config.emb.py b/demo/quick_start/trainer_config.emb.py index 34dd7b96f2f14..8fd18a7aac704 100644 --- a/demo/quick_start/trainer_config.emb.py +++ b/demo/quick_start/trainer_config.emb.py @@ -27,18 +27,16 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer() -) + batch_size=batch_size, learning_rate=2e-3, learning_method=AdamOptimizer()) data = data_layer(name="word", size=len(word_dict)) embedding = embedding_layer(input=data, size=128) diff --git a/demo/quick_start/trainer_config.lr.py b/demo/quick_start/trainer_config.lr.py index 119e3849a4b7e..b9c9441baac28 100644 --- a/demo/quick_start/trainer_config.lr.py +++ b/demo/quick_start/trainer_config.lr.py @@ -16,7 +16,7 @@ from paddle.trainer_config_helpers import * -dict_file = "./data/dict.txt" +dict_file = get_config_arg('dict_file', str, "./data/dict.txt") word_dict = dict() with open(dict_file, 'r') as f: for i, line in enumerate(f): @@ -32,11 +32,12 @@ # We need to use different process for training and prediction. # For training, the input data includes both word IDs and labels. # For prediction, the input data only includs word Ids. -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_bow", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_bow", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -44,8 +45,7 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) # Define the data for text features. The size of the data layer is the number # of words in the dictionary. @@ -63,7 +63,6 @@ label = data_layer(name="label", size=2) # Define cross-entropy classification loss and error. - classification_cost(input=output, label=label) cls = classification_cost(input=output, label=label) outputs(cls) else: diff --git a/demo/quick_start/trainer_config.lstm.py b/demo/quick_start/trainer_config.lstm.py index ec8a2cb00abd1..8821e02d9bd4a 100644 --- a/demo/quick_start/trainer_config.lstm.py +++ b/demo/quick_start/trainer_config.lstm.py @@ -27,11 +27,12 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,24 +40,14 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) - -bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) + gradient_clipping_threshold=25) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) -fc = fc_layer(input=emb, size=512, - act=LinearActivation(), - bias_attr=bias_attr, - layer_attr=ExtraAttr(drop_rate=0.1)) -lstm = lstmemory(input=fc, act=TanhActivation(), - bias_attr=bias_attr, - layer_attr=ExtraAttr(drop_rate=0.25)) -lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer(input=lstm_last, size=2, - bias_attr=bias_attr, - act=SoftmaxActivation()) +lstm = simple_lstm( + input=emb, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.25)) +lstm_max = pooling_layer(input=lstm, pooling_type=MaxPooling()) +output = fc_layer(input=lstm_max, size=2, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) outputs([maxid, output]) diff --git a/demo/quick_start/trainer_config.resnet-lstm.py b/demo/quick_start/trainer_config.resnet-lstm.py new file mode 100644 index 0000000000000..91e1581c386eb --- /dev/null +++ b/demo/quick_start/trainer_config.resnet-lstm.py @@ -0,0 +1,94 @@ +# edit-mode: -*- python -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This configuration is a demonstration of how to implement the stacked LSTM +with residual connections, i.e. an LSTM layer takes the sum of the hidden states +and inputs of the previous LSTM layer instead of only the hidden states. +This architecture is from: +Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V. Le, Mohammad Norouzi, +Wolfgang Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, +Jeff Klingner, Apurva Shah, Melvin Johnson, Xiaobing Liu, Lukasz Kaiser, +Stephan Gouws, Yoshikiyo Kato, Taku Kudo, Hideto Kazawa, Keith Stevens, +George Kurian, Nishant Patil, Wei Wang, Cliff Young, Jason Smith, Jason Riesa, +Alex Rudnick, Oriol Vinyals, Greg Corrado, Macduff Hughes, Jeffrey Dean. 2016. +Google's Neural Machine Translation System: Bridging the Gap between Human and +Machine Translation. In arXiv https://arxiv.org/pdf/1609.08144v2.pdf +Different from the architecture described in the paper, we use a stack single +direction LSTM layers as the first layer instead of bi-directional LSTM. Also, +since this is a demo code, to reduce computation time, we stacked 4 layers +instead of 8 layers. +""" + +from paddle.trainer_config_helpers import * + +dict_file = "./data/dict.txt" +word_dict = dict() +with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + +is_predict = get_config_arg('is_predict', bool, False) +trn = 'data/train.list' if not is_predict else None +tst = 'data/test.list' if not is_predict else 'data/pred.list' +process = 'process' if not is_predict else 'process_predict' +define_py_data_sources2(train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) + +batch_size = 128 if not is_predict else 1 +settings( + batch_size=batch_size, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25 +) + +bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) + +data = data_layer(name="word", size=len(word_dict)) +emb = embedding_layer(input=data, size=128) +lstm = simple_lstm(input=emb, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.1)) + +previous_input, previous_hidden_state = emb, lstm + +for i in range(3): + # The input to the current layer is the sum of the hidden state + # and input of the previous layer. + current_input = addto_layer(input=[previous_input, previous_hidden_state]) + hidden_state = simple_lstm(input=current_input, size=128, + lstm_cell_attr=ExtraAttr(drop_rate=0.1)) + previous_input, previous_hidden_state = current_input, hidden_state + +lstm = previous_hidden_state + +lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) +output = fc_layer(input=lstm_last, size=2, + bias_attr=bias_attr, + act=SoftmaxActivation()) + + +if is_predict: + maxid = maxid_layer(output) + outputs([maxid, output]) +else: + label = data_layer(name="label", size=2) + cls = classification_cost(input=output, label=label) + outputs(cls) diff --git a/demo/recommendation/common_utils.py b/demo/recommendation/common_utils.py index a5f00b3ef9ca0..613e36b496e47 100755 --- a/demo/recommendation/common_utils.py +++ b/demo/recommendation/common_utils.py @@ -21,8 +21,9 @@ def meta_to_header(meta, name): yield integer_value(each_meta['max']) elif each_meta['type'] == 'embedding': is_seq = each_meta['seq'] == 'sequence' - yield integer_value(len(each_meta['dict']), - seq_type=SequenceType.SEQUENCE if is_seq - else SequenceType.NO_SEQUENCE) + yield integer_value( + len(each_meta['dict']), + seq_type=SequenceType.SEQUENCE + if is_seq else SequenceType.NO_SEQUENCE) elif each_meta['type'] == 'one_hot_dense': yield dense_vector(len(each_meta['dict'])) diff --git a/demo/recommendation/data/config.json b/demo/recommendation/data/config.json index 71a9dd7be6bd1..f26e74ce47bb7 100644 --- a/demo/recommendation/data/config.json +++ b/demo/recommendation/data/config.json @@ -14,4 +14,3 @@ "fields": ["id", "title", "genres"] } } - diff --git a/demo/recommendation/data/config_generator.py b/demo/recommendation/data/config_generator.py index 29f38082693ad..fa605458300f8 100644 --- a/demo/recommendation/data/config_generator.py +++ b/demo/recommendation/data/config_generator.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ config_generator.py @@ -29,10 +28,7 @@ import docopt import copy -DEFAULT_FILE = { - "type": "split", - "delimiter": "," -} +DEFAULT_FILE = {"type": "split", "delimiter": ","} DEFAULT_FIELD = { "id": { @@ -107,19 +103,16 @@ def main(filename, fmt): field = copy.deepcopy(DEFAULT_FIELD[field_key]) field['pos'] = pos fields.append(field) - obj[k] = { - "file": file_dict, - "fields": fields - } - meta = { - "meta": obj - } + obj[k] = {"file": file_dict, "fields": fields} + meta = {"meta": obj} # print meta if fmt == 'json': + def formatter(x): import json return json.dumps(x, indent=2) elif fmt == 'yaml': + def formatter(x): import yaml return yaml.safe_dump(x, default_flow_style=False) diff --git a/demo/recommendation/data/meta_generator.py b/demo/recommendation/data/meta_generator.py index 8d1a33d02aea1..593c863670d5e 100644 --- a/demo/recommendation/data/meta_generator.py +++ b/demo/recommendation/data/meta_generator.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Preprocess Movielens dataset, to get movie/user object. @@ -66,8 +65,8 @@ def scan(self, key): self.__key_set__.add(key) def finish_scan(self, compare=None, key=None, reverse=False): - self.__key_set__ = sorted(list(self.__key_set__), cmp=compare, - key=key, reverse=reverse) + self.__key_set__ = sorted( + list(self.__key_set__), cmp=compare, key=key, reverse=reverse) self.dict = dict() for idx, each_key in enumerate(self.__key_set__): self.dict[each_key] = idx @@ -207,11 +206,10 @@ def __init__(self, config): self.dict = EmbeddingFieldParser.CharBasedEmbeddingDict( self.seq_type == EmbeddingFieldParser.SEQUENCE) elif config['dict']['type'] == 'split': - self.dict = SplitEmbeddingDict( - config['dict'].get('delimiter', ',')) + self.dict = SplitEmbeddingDict(config['dict'].get('delimiter', ',')) elif config['dict']['type'] == 'whole_content': - self.dict = EmbeddingFieldParser.WholeContentDict( - config['dict']['sort']) + self.dict = EmbeddingFieldParser.WholeContentDict(config['dict'][ + 'sort']) else: print config assert False @@ -333,8 +331,8 @@ def create(config): return PositionContentExtractor(config['pos']) else: extra_args = config['regex'] - return RegexPositionContentExtractor(pos=config['pos'], - **extra_args) + return RegexPositionContentExtractor( + pos=config['pos'], **extra_args) class MetaFile(object): @@ -364,9 +362,10 @@ def parse(self, config): metas = map(lambda x: x.meta_field(), field_parsers) # print metas - key_index = filter(lambda x: x is not None, map( - lambda (idx, meta): idx if 'is_key' in meta and meta['is_key'] - else None, enumerate(metas)))[0] + key_index = filter( + lambda x: x is not None, + map(lambda (idx, meta): idx if 'is_key' in meta and meta['is_key'] else None, + enumerate(metas)))[0] key_map = [] for i in range(min(key_index, len(metas))): @@ -374,12 +373,7 @@ def parse(self, config): for i in range(key_index + 1, len(metas)): key_map.append(i) - obj = { - '__meta__': { - 'raw_meta': metas, - 'feature_map': key_map - } - } + obj = {'__meta__': {'raw_meta': metas, 'feature_map': key_map}} for each_block in reader.read(): idx = field_parsers[key_index].parse(each_block) diff --git a/demo/recommendation/data/split.py b/demo/recommendation/data/split.py index ff1f7fab7befd..8dd0cbd32af60 100644 --- a/demo/recommendation/data/split.py +++ b/demo/recommendation/data/split.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Separate movielens 1m dataset to train/test file. diff --git a/demo/recommendation/dataprovider.py b/demo/recommendation/dataprovider.py index 454467f40b44b..ff3932be03f1e 100755 --- a/demo/recommendation/dataprovider.py +++ b/demo/recommendation/dataprovider.py @@ -15,6 +15,7 @@ from paddle.trainer.PyDataProvider2 import * import common_utils # parse + def hook(settings, meta, **kwargs): """ Init hook is invoked before process data. It will set obj.slots and store @@ -41,6 +42,7 @@ def hook(settings, meta, **kwargs): settings.input_types = headers settings.meta = meta + @provider(init_hook=hook, cache=CacheType.CACHE_PASS_IN_MEM) def process(settings, filename): with open(filename, 'r') as f: diff --git a/demo/recommendation/prediction.py b/demo/recommendation/prediction.py index f8044a3195ec2..e2a202cfd1a47 100755 --- a/demo/recommendation/prediction.py +++ b/demo/recommendation/prediction.py @@ -28,7 +28,8 @@ model_path = sys.argv[1] swig_paddle.initPaddle('--use_gpu=0') conf = parse_config("trainer_config.py", "is_predict=1") - network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) network.loadParameters(model_path) with open('./data/meta.bin', 'rb') as f: @@ -39,11 +40,12 @@ while True: movie_id = int(raw_input("Input movie_id: ")) user_id = int(raw_input("Input user_id: ")) - movie_meta = meta['movie'][movie_id] # Query Data From Meta. + movie_meta = meta['movie'][movie_id] # Query Data From Meta. user_meta = meta['user'][user_id] data = [movie_id - 1] data.extend(movie_meta) data.append(user_id - 1) data.extend(user_meta) - print "Prediction Score is %.2f" % ((network.forwardTest( - cvt.convert([data]))[0]['value'][0][0] + 5) / 2) + print "Prediction Score is %.2f" % ( + (network.forwardTest(cvt.convert([data]))[0]['value'][0][0] + 5) + / 2) diff --git a/demo/recommendation/trainer_config.py b/demo/recommendation/trainer_config.py index 624c22ec969dc..cec340b0b65a8 100755 --- a/demo/recommendation/trainer_config.py +++ b/demo/recommendation/trainer_config.py @@ -27,8 +27,8 @@ # load meta file meta = pickle.load(f) -settings(batch_size=1600, learning_rate=1e-3, - learning_method=RMSPropOptimizer()) +settings( + batch_size=1600, learning_rate=1e-3, learning_method=RMSPropOptimizer()) def construct_feature(name): @@ -59,11 +59,10 @@ def construct_feature(name): slot_name = each_meta.get('name', '%s_id' % name) if type_name == 'id': slot_dim = each_meta['max'] - embedding = embedding_layer(input=data_layer(slot_name, - size=slot_dim), - size=256) - fusion.append(fc_layer(input=embedding, - size=256)) + embedding = embedding_layer( + input=data_layer( + slot_name, size=slot_dim), size=256) + fusion.append(fc_layer(input=embedding, size=256)) elif type_name == 'embedding': is_seq = each_meta['seq'] == 'sequence' slot_dim = len(each_meta['dict']) @@ -71,17 +70,14 @@ def construct_feature(name): embedding = embedding_layer(input=din, size=256) if is_seq: fusion.append( - text_conv_pool(input=embedding, context_len=5, - hidden_size=256)) + text_conv_pool( + input=embedding, context_len=5, hidden_size=256)) else: - fusion.append(fc_layer(input=embedding, - size=256)) + fusion.append(fc_layer(input=embedding, size=256)) elif type_name == 'one_hot_dense': slot_dim = len(each_meta['dict']) - hidden = fc_layer(input=data_layer(slot_name, slot_dim), - size=256) - fusion.append(fc_layer(input=hidden, - size=256)) + hidden = fc_layer(input=data_layer(slot_name, slot_dim), size=256) + fusion.append(fc_layer(input=hidden, size=256)) return fc_layer(name="%s_fusion" % name, input=fusion, size=256) @@ -90,10 +86,16 @@ def construct_feature(name): user_feature = construct_feature("user") similarity = cos_sim(a=movie_feature, b=user_feature) if not is_predict: - outputs(regression_cost(input=similarity, - label=data_layer('rating', size=1))) - - define_py_data_sources2('data/train.list', 'data/test.list', module='dataprovider', - obj='process', args={'meta': meta}) + outputs( + regression_cost( + input=similarity, label=data_layer( + 'rating', size=1))) + + define_py_data_sources2( + 'data/train.list', + 'data/test.list', + module='dataprovider', + obj='process', + args={'meta': meta}) else: outputs(similarity) diff --git a/demo/semantic_role_labeling/.gitignore b/demo/semantic_role_labeling/.gitignore new file mode 100644 index 0000000000000..cd90ca7bbe9be --- /dev/null +++ b/demo/semantic_role_labeling/.gitignore @@ -0,0 +1,10 @@ +*.pyc +train.log +data/feature +data/conll05st-release/ +data/src.dict +data/test.wsj.props +data/test.wsj.seq_pair +data/test.wsj.words +data/tgt.dict +output diff --git a/demo/semantic_role_labeling/dataprovider.py b/demo/semantic_role_labeling/dataprovider.py index 2ef25c42c1794..5c003584a52d4 100644 --- a/demo/semantic_role_labeling/dataprovider.py +++ b/demo/semantic_role_labeling/dataprovider.py @@ -26,9 +26,9 @@ def hook(settings, word_dict, label_dict, **kwargs): integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(2), - integer_value_sequence(len(label_dict))] + integer_value_sequence(len(word_dict)), integer_value_sequence(2), + integer_value_sequence(len(label_dict)) + ] @provider(init_hook=hook) diff --git a/demo/semantic_role_labeling/db_lstm.py b/demo/semantic_role_labeling/db_lstm.py index 364460afbe31c..e3f6edad69721 100644 --- a/demo/semantic_role_labeling/db_lstm.py +++ b/demo/semantic_role_labeling/db_lstm.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import math import os import sys @@ -42,7 +41,7 @@ label_dict[w] = i if is_test: - train_list_file = None + train_list_file = None #define data provider define_py_data_sources2( diff --git a/demo/semantic_role_labeling/predict.py b/demo/semantic_role_labeling/predict.py index 9a27112828e44..f051d4175cf6f 100644 --- a/demo/semantic_role_labeling/predict.py +++ b/demo/semantic_role_labeling/predict.py @@ -41,22 +41,16 @@ def __init__(self, train_conf, dict_file, model_dir, label_file): len_dict = len(self.dict) len_label = len(self.labels) - conf = parse_config( - train_conf, - 'dict_len=' + str(len_dict) + - ',label_len=' + str(len_label) + - ',is_predict=True') + conf = parse_config(train_conf, 'dict_len=' + str(len_dict) + + ',label_len=' + str(len_label) + ',is_predict=True') self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) self.network.loadParameters(model_dir) slots = [ - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(2) + integer_value_sequence(len_dict), integer_value_sequence(len_dict), + integer_value_sequence(len_dict), integer_value_sequence(len_dict), + integer_value_sequence(len_dict), integer_value_sequence(2) ] self.converter = DataProviderConverter(slots) @@ -110,8 +104,8 @@ def predict(self, data_file): len_sen = len(sen.split()) line_labels = lab[index:index + len_sen] index += len_sen - fout.write(sen + '\t' + ' '.join([self.labels_reverse[ - i] for i in line_labels]) + '\n') + fout.write(sen + '\t' + ' '.join( + [self.labels_reverse[i] for i in line_labels]) + '\n') def option_parser(): diff --git a/demo/semantic_role_labeling/predict.sh b/demo/semantic_role_labeling/predict.sh index a545b9a5d591b..111eea95a22d3 100644 --- a/demo/semantic_role_labeling/predict.sh +++ b/demo/semantic_role_labeling/predict.sh @@ -18,7 +18,7 @@ set -e function get_best_pass() { cat $1 | grep -Pzo 'Test .*\n.*pass-.*' | \ sed -r 'N;s/Test.* cost=([0-9]+\.[0-9]+).*\n.*pass-([0-9]+)/\1 \2/g' | \ - sort | head -n 1 + sort -n | head -n 1 } log=train.log @@ -26,7 +26,6 @@ LOG=`get_best_pass $log` LOG=(${LOG}) best_model_path="output/pass-${LOG[1]}" - config_file=db_lstm.py dict_file=./data/src.dict label_file=./data/tgt.dict diff --git a/demo/semantic_role_labeling/test.sh b/demo/semantic_role_labeling/test.sh index 804f722e5b8e9..f9e1bdcd4c752 100644 --- a/demo/semantic_role_labeling/test.sh +++ b/demo/semantic_role_labeling/test.sh @@ -18,7 +18,7 @@ set -e function get_best_pass() { cat $1 | grep -Pzo 'Test .*\n.*pass-.*' | \ sed -r 'N;s/Test.* cost=([0-9]+\.[0-9]+).*\n.*pass-([0-9]+)/\1 \2/g' |\ - sort | head -n 1 + sort -n | head -n 1 } log=train.log @@ -36,5 +36,5 @@ paddle train \ --job=test \ --use_gpu=false \ --config_args=is_test=1 \ + --test_all_data_in_one_period=1 \ 2>&1 | tee 'test.log' - diff --git a/demo/semantic_role_labeling/train.sh b/demo/semantic_role_labeling/train.sh index 94c7b6f31df3b..c3a22b644be0c 100644 --- a/demo/semantic_role_labeling/train.sh +++ b/demo/semantic_role_labeling/train.sh @@ -24,4 +24,3 @@ paddle train \ --show_parameter_stats_period=10 \ --test_all_data_in_one_period=1 \ 2>&1 | tee 'train.log' - diff --git a/demo/sentiment/dataprovider.py b/demo/sentiment/dataprovider.py index 9a9fd81f030cb..53e3d1d20df92 100755 --- a/demo/sentiment/dataprovider.py +++ b/demo/sentiment/dataprovider.py @@ -17,8 +17,8 @@ def hook(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ - integer_value_sequence(len(settings.word_dict)), - integer_value(2)] + integer_value_sequence(len(settings.word_dict)), integer_value(2) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -29,6 +29,7 @@ def process(settings, file_name): label, comment = line.strip().split('\t\t') label = int(label) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if w in - settings.word_dict] + word_slot = [ + settings.word_dict[w] for w in words if w in settings.word_dict + ] yield word_slot, label diff --git a/demo/sentiment/predict.py b/demo/sentiment/predict.py index c61628d34db4a..bc0f6f3126429 100755 --- a/demo/sentiment/predict.py +++ b/demo/sentiment/predict.py @@ -18,14 +18,14 @@ from py_paddle import swig_paddle, DataProviderConverter from paddle.trainer.PyDataProvider2 import integer_value_sequence from paddle.trainer.config_parser import parse_config - """ Usage: run following command to show help message. python predict.py -h """ + class SentimentPrediction(): - def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): + def __init__(self, train_conf, dict_file, model_dir=None, label_file=None): """ train_conf: trainer configure. dict_file: word dictionary file name. @@ -44,10 +44,11 @@ def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): self.load_label(label_file) conf = parse_config(train_conf, "is_predict=1") - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) self.network.loadParameters(self.model_dir) - slots = [integer_value_sequence(self.dict_dim)] - self.converter = DataProviderConverter(slots) + input_types = [integer_value_sequence(self.dict_dim)] + self.converter = DataProviderConverter(input_types) def load_dict(self): """ @@ -61,7 +62,7 @@ def load_label(self, label_file): """ Load label. """ - self.label={} + self.label = {} for v in open(label_file, 'r'): self.label[int(v.split('\t')[1])] = v.split('\t')[0] @@ -72,7 +73,9 @@ def get_data(self, data_file): with open(data_file, 'r') as fdata: for line in fdata: words = line.strip().split() - word_slot = [self.word_dict[w] for w in words if w in self.word_dict] + word_slot = [ + self.word_dict[w] for w in words if w in self.word_dict + ] if not word_slot: print "all words are not in dictionary: %s", line continue @@ -89,25 +92,48 @@ def predict(self, data_file): if self.label is None: print("%s: predicting label is %d" % (data_file, lab[0][0])) else: - print("%s: predicting label is %s" % (data_file, self.label[lab[0][0]])) + print("%s: predicting label is %s" % + (data_file, self.label[lab[0][0]])) + def option_parser(): usage = "python predict.py -n config -w model_dir -d dictionary -i input_file " parser = OptionParser(usage="usage: %s [options]" % usage) - parser.add_option("-n", "--tconf", action="store", - dest="train_conf", help="network config") - parser.add_option("-d", "--dict", action="store", - dest="dict_file",help="dictionary file") - parser.add_option("-b", "--label", action="store", - dest="label", default=None, - help="dictionary file") - parser.add_option("-i", "--data", action="store", - dest="data", help="data file to predict") - parser.add_option("-w", "--model", action="store", - dest="model_path", default=None, - help="model path") + parser.add_option( + "-n", + "--tconf", + action="store", + dest="train_conf", + help="network config") + parser.add_option( + "-d", + "--dict", + action="store", + dest="dict_file", + help="dictionary file") + parser.add_option( + "-b", + "--label", + action="store", + dest="label", + default=None, + help="dictionary file") + parser.add_option( + "-i", + "--data", + action="store", + dest="data", + help="data file to predict") + parser.add_option( + "-w", + "--model", + action="store", + dest="model_path", + default=None, + help="model path") return parser.parse_args() + def main(): options, args = option_parser() train_conf = options.train_conf @@ -119,5 +145,6 @@ def main(): predict = SentimentPrediction(train_conf, dict_file, model_path, label) predict.predict(data) + if __name__ == '__main__': main() diff --git a/demo/sentiment/preprocess.py b/demo/sentiment/preprocess.py index 49b53d500a1bf..7146e95d751c4 100755 --- a/demo/sentiment/preprocess.py +++ b/demo/sentiment/preprocess.py @@ -22,13 +22,13 @@ from optparse import OptionParser from paddle.utils.preprocess_util import * - """ Usage: run following command to show help message. python preprocess.py -h """ -def save_dict(dict, filename, is_reverse = True): + +def save_dict(dict, filename, is_reverse=True): """ Save dictionary into file. dict: input dictionary. @@ -39,9 +39,10 @@ def save_dict(dict, filename, is_reverse = True): f = open(filename, 'w') for k, v in sorted(dict.items(), key=operator.itemgetter(1),\ reverse=is_reverse): - f.write('%s\t%s\n'%(k, v)) + f.write('%s\t%s\n' % (k, v)) f.close() + def tokenize(sentences): """ Use tokenizer.perl to tokenize input sentences. @@ -58,6 +59,7 @@ def tokenize(sentences): toks = tok_text.split('\n')[:-1] return toks + def read_lines(path): """ path: String, file path. @@ -71,12 +73,17 @@ def read_lines(path): seqs.append(line) return seqs + class SentimentDataSetCreate(): """ A class to process data for sentiment analysis task. """ - def __init__(self, data_path, output_path, - use_okenizer = True, multi_lines = False): + + def __init__(self, + data_path, + output_path, + use_okenizer=True, + multi_lines=False): """ data_path: string, traing and testing dataset path output_path: string, output path, store processed dataset @@ -164,23 +171,17 @@ def create_dataset(self): # Preprocess train data. train_data, train_lab_set = self.data_list(self.train_dir) print "processing train set..." - file_lists = self.save_data(train_data, - "train", - self.batch_size, - True, - True) + file_lists = self.save_data(train_data, "train", self.batch_size, True, + True) save_list(file_lists, self.train_list) # If have test data path, preprocess test data. if os.path.exists(self.test_dir): test_data, test_lab_set = self.data_list(self.test_dir) - assert(train_lab_set == test_lab_set) + assert (train_lab_set == test_lab_set) print "processing test set..." - file_lists = self.save_data(test_data, - "test", - self.batch_size, - False, - self.dict_with_test) + file_lists = self.save_data(test_data, "test", self.batch_size, + False, self.dict_with_test) save_list(file_lists, self.test_list) # save labels set. @@ -191,7 +192,9 @@ def create_dataset(self): save_dict(self.word_count, self.dict_file, True) self.dict_size = len(self.word_count) - def save_data(self, data, prefix = "", + def save_data(self, + data, + prefix="", batch_size=50000, is_shuffle=False, build_dict=False): @@ -205,7 +208,8 @@ def save_data(self, data, prefix = "", return: list of batch names """ if is_shuffle and self.multi_lines: - return self.save_data_multi_lines(data, prefix, batch_size, build_dict) + return self.save_data_multi_lines(data, prefix, batch_size, + build_dict) if is_shuffle: random.shuffle(data) @@ -213,7 +217,7 @@ def save_data(self, data, prefix = "", batch_names = [] for i in range(num_batches): batch_name = join_path(self.output_path, - "%s_part_%03d" %(prefix, i)) + "%s_part_%03d" % (prefix, i)) begin = i * batch_size end = min((i + 1) * batch_size, len(data)) # read a batch of data @@ -246,7 +250,9 @@ def get_data_list(self, begin, end, data): data_list = tokenize(data_list) return label_list, data_list - def save_data_multi_lines(self, data, prefix = "", + def save_data_multi_lines(self, + data, + prefix="", batch_size=50000, build_dict=False): """ @@ -274,14 +280,14 @@ def save_data_multi_lines(self, data, prefix = "", self.create_dict(data_list) length = len(label_list) - perm_list = np.array([ i for i in xrange(length) ]) + perm_list = np.array([i for i in xrange(length)]) random.shuffle(perm_list) num_batches = int(math.ceil(length / float(batch_size))) batch_names = [] for i in range(num_batches): batch_name = join_path(self.output_path, - "%s_part_%03d" %(prefix, i)) + "%s_part_%03d" % (prefix, i)) begin = i * batch_size end = min((i + 1) * batch_size, length) sub_label = [label_list[perm_list[i]] for i in range(begin, end)] @@ -304,35 +310,50 @@ def save_file(self, label_list, data_list, filename): f.write('%s\t\t%s\n' % (lab, seq)) f.close() + def option_parser(): parser = OptionParser(usage="usage: python preprcoess.py "\ "-i data_dir [options]") - parser.add_option("-i", "--data", action="store", - dest="input", help="Input data directory.") - parser.add_option("-o", "--output", action="store", - dest="output", default=None, - help="Output directory.") - parser.add_option("-t", "--tokenizer", action="store", - dest="use_tokenizer", default=True, - help="Whether to use tokenizer.") + parser.add_option( + "-i", + "--data", + action="store", + dest="input", + help="Input data directory.") + parser.add_option( + "-o", + "--output", + action="store", + dest="output", + default=None, + help="Output directory.") + parser.add_option( + "-t", + "--tokenizer", + action="store", + dest="use_tokenizer", + default=True, + help="Whether to use tokenizer.") parser.add_option("-m", "--multi_lines", action="store", dest="multi_lines", default=False, help="If input text files have multi lines and they "\ "need to be shuffled, you should set -m True,") return parser.parse_args() + def main(): options, args = option_parser() - data_dir=options.input - output_dir=options.output - use_tokenizer=options.use_tokenizer - multi_lines=options.multi_lines + data_dir = options.input + output_dir = options.output + use_tokenizer = options.use_tokenizer + multi_lines = options.multi_lines if output_dir is None: outname = os.path.basename(options.input) output_dir = join_path(os.path.dirname(data_dir), 'pre-' + outname) - data_creator = SentimentDataSetCreate(data_dir, output_dir, - use_tokenizer, multi_lines) + data_creator = SentimentDataSetCreate(data_dir, output_dir, use_tokenizer, + multi_lines) data_creator.create_dataset() + if __name__ == '__main__': main() diff --git a/demo/sentiment/sentiment_net.py b/demo/sentiment/sentiment_net.py index 31e585edcaa11..ff6a3624a404c 100644 --- a/demo/sentiment/sentiment_net.py +++ b/demo/sentiment/sentiment_net.py @@ -47,10 +47,12 @@ def sentiment_data(data_dir=None, for i, line in enumerate(open(dict_file, 'r')): word_dict[line.split('\t')[0]] = i - define_py_data_sources2(train_list, test_list, - module="dataprovider", - obj="process", - args={'dictionary': word_dict}) + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={'dictionary': word_dict}) return dict_dim, class_dim @@ -64,8 +66,7 @@ def bidirectional_lstm_net(input_dim, emb = embedding_layer(input=data, size=emb_dim) bi_lstm = bidirectional_lstm(input=emb, size=lstm_dim) dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) - output = fc_layer(input=dropout, size=class_dim, - act=SoftmaxActivation()) + output = fc_layer(input=dropout, size=class_dim, act=SoftmaxActivation()) if not is_predict: lbl = data_layer("label", 1) @@ -109,27 +110,36 @@ def stacked_lstm_net(input_dim, data = data_layer("word", input_dim) emb = embedding_layer(input=data, size=emb_dim) - fc1 = fc_layer(input=emb, size=hid_dim, act=linear, - bias_attr=bias_attr) - lstm1 = lstmemory(input=fc1, act=relu, bias_attr=bias_attr, - layer_attr=layer_attr) + fc1 = fc_layer(input=emb, size=hid_dim, act=linear, bias_attr=bias_attr) + lstm1 = lstmemory( + input=fc1, act=relu, bias_attr=bias_attr, layer_attr=layer_attr) inputs = [fc1, lstm1] for i in range(2, stacked_num + 1): - fc = fc_layer(input=inputs, size=hid_dim, act=linear, - param_attr=para_attr, bias_attr=bias_attr) - lstm = lstmemory(input=fc, reverse=(i % 2) == 0, act=relu, - bias_attr=bias_attr, layer_attr=layer_attr) + fc = fc_layer( + input=inputs, + size=hid_dim, + act=linear, + param_attr=para_attr, + bias_attr=bias_attr) + lstm = lstmemory( + input=fc, + reverse=(i % 2) == 0, + act=relu, + bias_attr=bias_attr, + layer_attr=layer_attr) inputs = [fc, lstm] fc_last = pooling_layer(input=inputs[0], pooling_type=MaxPooling()) lstm_last = pooling_layer(input=inputs[1], pooling_type=MaxPooling()) - output = fc_layer(input=[fc_last, lstm_last], size=class_dim, - act=SoftmaxActivation(), - bias_attr=bias_attr, param_attr=para_attr) + output = fc_layer( + input=[fc_last, lstm_last], + size=class_dim, + act=SoftmaxActivation(), + bias_attr=bias_attr, + param_attr=para_attr) if is_predict: outputs(output) else: - outputs( - classification_cost(input=output, label=data_layer('label', 1))) + outputs(classification_cost(input=output, label=data_layer('label', 1))) diff --git a/demo/sentiment/test.sh b/demo/sentiment/test.sh index 098fbb91389b8..c8b12a0e89dbd 100755 --- a/demo/sentiment/test.sh +++ b/demo/sentiment/test.sh @@ -17,7 +17,7 @@ set -e function get_best_pass() { cat $1 | grep -Pzo 'Test .*\n.*pass-.*' | \ sed -r 'N;s/Test.* classification_error_evaluator=([0-9]+\.[0-9]+).*\n.*pass-([0-9]+)/\1 \2/g' |\ - sort | head -n 1 + sort -n | head -n 1 } log=train.log diff --git a/demo/sentiment/trainer_config.py b/demo/sentiment/trainer_config.py index db24182a8d735..894070e7c97dc 100644 --- a/demo/sentiment/trainer_config.py +++ b/demo/sentiment/trainer_config.py @@ -20,20 +20,19 @@ # whether this config is used for prediction is_predict = get_config_arg('is_predict', bool, False) -data_dir = "./data/pre-imdb" +data_dir = "./data/pre-imdb" dict_dim, class_dim = sentiment_data(data_dir, is_test, is_predict) ################## Algorithm Config ##################### settings( - batch_size=128, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + batch_size=128, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25) #################### Network Config ###################### -stacked_lstm_net(dict_dim, class_dim=class_dim, - stacked_num=3, is_predict=is_predict) +stacked_lstm_net( + dict_dim, class_dim=class_dim, stacked_num=3, is_predict=is_predict) # bidirectional_lstm_net(dict_dim, class_dim=class_dim, is_predict=is_predict) diff --git a/demo/seqToseq/data/paraphrase_data.sh b/demo/seqToseq/data/paraphrase_data.sh index ea1f8dbcfad35..1b3f1d45e11fb 100755 --- a/demo/seqToseq/data/paraphrase_data.sh +++ b/demo/seqToseq/data/paraphrase_data.sh @@ -16,9 +16,7 @@ set -e set -x # download the in-house paraphrase dataset -# following is the google drive address -# you can also directly download from https://pan.baidu.com/s/1o8q577s -wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/embedding/paraphrase.tar.gz --no-check-certificate +wget http://paddlepaddle.bj.bcebos.com/model_zoo/embedding/paraphrase.tar.gz # untar the dataset tar -zxvf paraphrase.tar.gz diff --git a/demo/seqToseq/data/wmt14_model.sh b/demo/seqToseq/data/wmt14_model.sh index 2cec30688d27a..d6e7a732644dc 100755 --- a/demo/seqToseq/data/wmt14_model.sh +++ b/demo/seqToseq/data/wmt14_model.sh @@ -16,9 +16,7 @@ set -e set -x # download the pretrained model -# following is the google drive address -# you can also directly download from https://pan.baidu.com/s/1o8q577s -wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/wmt14_model.tar.gz --no-check-certificate +wget http://paddlepaddle.bj.bcebos.com/model_zoo/wmt14_model.tar.gz # untar the model tar -zxvf wmt14_model.tar.gz diff --git a/demo/seqToseq/dataprovider.py b/demo/seqToseq/dataprovider.py index df19db109ed22..c5da1b7685f47 100755 --- a/demo/seqToseq/dataprovider.py +++ b/demo/seqToseq/dataprovider.py @@ -30,14 +30,14 @@ def hook(settings, src_dict, trg_dict, file_list, **kwargs): if settings.job_mode: settings.trg_dict = trg_dict settings.slots = [ - integer_value_sequence(len(settings.src_dict)), - integer_value_sequence(len(settings.trg_dict)), + integer_value_sequence(len(settings.src_dict)), + integer_value_sequence(len(settings.trg_dict)), integer_value_sequence(len(settings.trg_dict)) ] settings.logger.info("trg dict len : %d" % (len(settings.trg_dict))) else: settings.slots = [ - integer_value_sequence(len(settings.src_dict)), + integer_value_sequence(len(settings.src_dict)), integer_value_sequence(len(open(file_list[0], "r").readlines())) ] @@ -62,8 +62,7 @@ def process(settings, file_name): if settings.job_mode: trg_seq = line_split[1] # one target sequence trg_words = trg_seq.split() - trg_ids = [settings.trg_dict.get(w, UNK_IDX) - for w in trg_words] + trg_ids = [settings.trg_dict.get(w, UNK_IDX) for w in trg_words] # remove sequence whose length > 80 in training mode if len(src_ids) > 80 or len(trg_ids) > 80: diff --git a/demo/seqToseq/preprocess.py b/demo/seqToseq/preprocess.py index 5efb17a664b9a..bd1c51b1514b7 100755 --- a/demo/seqToseq/preprocess.py +++ b/demo/seqToseq/preprocess.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python preprocess.py -i INPUT [-d DICTSIZE] [-m] @@ -24,12 +23,13 @@ -m --mergeDict merge source and target dictionary """ import os -import sys +import sys import string from optparse import OptionParser from paddle.utils.preprocess_util import save_list, DatasetCreater + class SeqToSeqDatasetCreater(DatasetCreater): """ A class to process data for sequence to sequence application. @@ -75,7 +75,7 @@ def cat_file(self, dir_path, suffix, output_path, output): if not os.path.exists(output): os.system(cmd + '> ' + output) - def build_dict(self, file_path, dict_path, dict_size = -1): + def build_dict(self, file_path, dict_path, dict_size=-1): """ Create the dictionary for the file, Note that 1. Valid characters include all printable characters @@ -99,20 +99,23 @@ def build_dict(self, file_path, dict_path, dict_size = -1): for word in words: if word not in dictory: dictory[word] = 1 - else: + else: dictory[word] += 1 output = open(dict_path, "w+") output.write('\n\n\n') count = 3 - for key, value in sorted(dictory.items(), key = lambda d:d[1], reverse = True): + for key, value in sorted( + dictory.items(), key=lambda d: d[1], reverse=True): output.write(key + "\n") count += 1 if count == dict_size: break self.dict_size = count - - def create_dataset(self, dict_size = -1, mergeDict = False, - suffixes = ['.src', '.trg']): + + def create_dataset(self, + dict_size=-1, + mergeDict=False, + suffixes=['.src', '.trg']): """ Create seqToseq dataset """ @@ -135,13 +138,14 @@ def create_dataset(self, dict_size = -1, mergeDict = False, # checkout dataset should be parallel corpora suffix_len = len(suffixes[0]) for dataset in dataset_list: - file_list = os.listdir(dataset) - if len(file_list) % 2 == 1: - raise RuntimeError("dataset should be parallel corpora") - file_list.sort() - for i in range(0, len(file_list), 2): - if file_list[i][:-suffix_len] != file_list[i + 1][:-suffix_len]: - raise RuntimeError("source and target file name should be equal") + file_list = os.listdir(dataset) + if len(file_list) % 2 == 1: + raise RuntimeError("dataset should be parallel corpora") + file_list.sort() + for i in range(0, len(file_list), 2): + if file_list[i][:-suffix_len] != file_list[i + 1][:-suffix_len]: + raise RuntimeError( + "source and target file name should be equal") # cat all the files with the same suffix in dataset for suffix in suffixes: @@ -155,16 +159,18 @@ def create_dataset(self, dict_size = -1, mergeDict = False, list = ['train.list', 'test.list', 'gen.list'] for dataset in dataset_list: outname = os.path.basename(dataset) - self.concat_file(dataset, outname + suffixes[0], + self.concat_file(dataset, outname + suffixes[0], outname + suffixes[1], dir_list[id], outname) - save_list([os.path.join(dir_list[id], outname)], + save_list([os.path.join(dir_list[id], outname)], os.path.join(self.output_path, list[id])) id += 1 # build dictionary for train data dict = ['src.dict', 'trg.dict'] - dict_path = [os.path.join(self.output_path, dict[0]), - os.path.join(self.output_path, dict[1])] + dict_path = [ + os.path.join(self.output_path, dict[0]), + os.path.join(self.output_path, dict[1]) + ] if mergeDict: outname = os.path.join(train_dir, train_dataset.split('/')[-1]) print 'build src dictionary for train data' @@ -173,22 +179,30 @@ def create_dataset(self, dict_size = -1, mergeDict = False, os.system('cp ' + dict_path[0] + ' ' + dict_path[1]) else: outname = os.path.join(train_dataset, self.train_dir_name) - for id in range(0,2): + for id in range(0, 2): suffix = suffixes[id] print 'build ' + suffix[1:] + ' dictionary for train data' self.build_dict(outname + suffix, dict_path[id], dict_size) print 'dictionary size is', self.dict_size + def main(): usage = "usage: \n" \ "python %prog -i INPUT [-d DICTSIZE] [-m]" parser = OptionParser(usage) - parser.add_option("-i", action="store", dest="input", - help="input original dataset path") - parser.add_option("-d", action="store", dest="dictsize", - help="specified word count of dictionary") - parser.add_option("-m", "--mergeDict", action="store_true", dest="mergeDict", - help="merge source and target dictionary") + parser.add_option( + "-i", action="store", dest="input", help="input original dataset path") + parser.add_option( + "-d", + action="store", + dest="dictsize", + help="specified word count of dictionary") + parser.add_option( + "-m", + "--mergeDict", + action="store_true", + dest="mergeDict", + help="merge source and target dictionary") (options, args) = parser.parse_args() if options.input[-1] == os.path.sep: options.input = options.input[:-1] @@ -200,5 +214,6 @@ def main(): data_creator = SeqToSeqDatasetCreater(options.input, output_path) data_creator.create_dataset(dictsize, options.mergeDict) + if __name__ == "__main__": - main(); + main() diff --git a/demo/seqToseq/seqToseq_net.py b/demo/seqToseq/seqToseq_net.py index 2b0c3f34648b0..ad5e3339c1461 100644 --- a/demo/seqToseq/seqToseq_net.py +++ b/demo/seqToseq/seqToseq_net.py @@ -50,16 +50,21 @@ def seq_to_seq_data(data_dir, trg_dict = None else: train_list = os.path.join(data_dir, train_list) - test_list = os.path.join(data_dir,test_list) + test_list = os.path.join(data_dir, test_list) - define_py_data_sources2(train_list, test_list, - module = "dataprovider", - obj = "process", - args = {"src_dict": src_dict, - "trg_dict": trg_dict}) + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={"src_dict": src_dict, + "trg_dict": trg_dict}) - return {"src_dict_path": src_lang_dict, "trg_dict_path": trg_lang_dict, - "gen_result": gen_result} + return { + "src_dict_path": src_lang_dict, + "trg_dict_path": trg_lang_dict, + "gen_result": gen_result + } def gru_encoder_decoder(data_conf, @@ -90,51 +95,55 @@ def gru_encoder_decoder(data_conf, size=word_vector_dim, param_attr=ParamAttr(name='_source_language_embedding')) src_forward = simple_gru(input=src_embedding, size=encoder_size) - src_backward = simple_gru(input=src_embedding, - size=encoder_size, - reverse=True) + src_backward = simple_gru( + input=src_embedding, size=encoder_size, reverse=True) encoded_vector = concat_layer(input=[src_forward, src_backward]) with mixed_layer(size=decoder_size) as encoded_proj: - encoded_proj += full_matrix_projection(encoded_vector) + encoded_proj += full_matrix_projection(input=encoded_vector) backward_first = first_seq(input=src_backward) - with mixed_layer(size=decoder_size, - act=TanhActivation(), ) as decoder_boot: - decoder_boot += full_matrix_projection(backward_first) + with mixed_layer( + size=decoder_size, + act=TanhActivation(), ) as decoder_boot: + decoder_boot += full_matrix_projection(input=backward_first) def gru_decoder_with_attention(enc_vec, enc_proj, current_word): - decoder_mem = memory(name='gru_decoder', - size=decoder_size, - boot_layer=decoder_boot) + decoder_mem = memory( + name='gru_decoder', size=decoder_size, boot_layer=decoder_boot) - context = simple_attention(encoded_sequence=enc_vec, - encoded_proj=enc_proj, - decoder_state=decoder_mem, ) + context = simple_attention( + encoded_sequence=enc_vec, + encoded_proj=enc_proj, + decoder_state=decoder_mem, ) with mixed_layer(size=decoder_size * 3) as decoder_inputs: - decoder_inputs += full_matrix_projection(context) - decoder_inputs += full_matrix_projection(current_word) - - gru_step = gru_step_layer(name='gru_decoder', - input=decoder_inputs, - output_mem=decoder_mem, - size=decoder_size) - - with mixed_layer(size=target_dict_dim, - bias_attr=True, - act=SoftmaxActivation()) as out: + decoder_inputs += full_matrix_projection(input=context) + decoder_inputs += full_matrix_projection(input=current_word) + + gru_step = gru_step_layer( + name='gru_decoder', + input=decoder_inputs, + output_mem=decoder_mem, + size=decoder_size) + + with mixed_layer( + size=target_dict_dim, bias_attr=True, + act=SoftmaxActivation()) as out: out += full_matrix_projection(input=gru_step) return out decoder_group_name = "decoder_group" - group_inputs=[StaticInput(input=encoded_vector,is_seq=True), - StaticInput(input=encoded_proj,is_seq=True)] + group_inputs = [ + StaticInput( + input=encoded_vector, is_seq=True), StaticInput( + input=encoded_proj, is_seq=True) + ] if not is_generating: trg_embedding = embedding_layer( - input=data_layer(name='target_language_word', - size=target_dict_dim), + input=data_layer( + name='target_language_word', size=target_dict_dim), size=word_vector_dim, param_attr=ParamAttr(name='_target_language_embedding')) group_inputs.append(trg_embedding) @@ -144,12 +153,12 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): # while encoded source sequence is accessed to as an unbounded memory. # Here, the StaticInput defines a read-only memory # for the recurrent_group. - decoder = recurrent_group(name=decoder_group_name, - step=gru_decoder_with_attention, - input=group_inputs) + decoder = recurrent_group( + name=decoder_group_name, + step=gru_decoder_with_attention, + input=group_inputs) - lbl = data_layer(name='target_language_next_word', - size=target_dict_dim) + lbl = data_layer(name='target_language_next_word', size=target_dict_dim) cost = classification_cost(input=decoder, label=lbl) outputs(cost) else: @@ -168,16 +177,19 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): embedding_size=word_vector_dim) group_inputs.append(trg_embedding) - beam_gen = beam_search(name=decoder_group_name, - step=gru_decoder_with_attention, - input=group_inputs, - bos_id=0, - eos_id=1, - beam_size=beam_size, - max_length=max_length) - - seqtext_printer_evaluator(input=beam_gen, - id_input=data_layer(name="sent_id", size=1), - dict_file=trg_dict_path, - result_file=gen_trans_file) + beam_gen = beam_search( + name=decoder_group_name, + step=gru_decoder_with_attention, + input=group_inputs, + bos_id=0, + eos_id=1, + beam_size=beam_size, + max_length=max_length) + + seqtext_printer_evaluator( + input=beam_gen, + id_input=data_layer( + name="sent_id", size=1), + dict_file=trg_dict_path, + result_file=gen_trans_file) outputs(beam_gen) diff --git a/demo/sequence_tagging/data/get_data.sh b/demo/sequence_tagging/data/get_data.sh new file mode 100755 index 0000000000000..e579d6c46ce5e --- /dev/null +++ b/demo/sequence_tagging/data/get_data.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +DIR="$( cd "$(dirname "$0")" ; pwd -P )" +cd $DIR + +wget http://www.cnts.ua.ac.be/conll2000/chunking/train.txt.gz +wget http://www.cnts.ua.ac.be/conll2000/chunking/test.txt.gz diff --git a/demo/sequence_tagging/data/test.list b/demo/sequence_tagging/data/test.list new file mode 100644 index 0000000000000..073c0a0c9063a --- /dev/null +++ b/demo/sequence_tagging/data/test.list @@ -0,0 +1 @@ +data/test.txt.gz diff --git a/demo/sequence_tagging/data/train.list b/demo/sequence_tagging/data/train.list new file mode 100644 index 0000000000000..43c24d5f6484a --- /dev/null +++ b/demo/sequence_tagging/data/train.list @@ -0,0 +1 @@ +data/train.txt.gz diff --git a/demo/sequence_tagging/dataprovider.py b/demo/sequence_tagging/dataprovider.py new file mode 100644 index 0000000000000..37dcb7aa17c0a --- /dev/null +++ b/demo/sequence_tagging/dataprovider.py @@ -0,0 +1,260 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.PyDataProvider2 import * +import gzip +import logging + +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) +logger = logging.getLogger('paddle') +logger.setLevel(logging.INFO) + +OOV_POLICY_IGNORE = 0 +OOV_POLICY_USE = 1 +OOV_POLICY_ERROR = 2 + +num_original_columns = 3 + +# Feature combination patterns. +# [[-1,0], [0,0]] means previous token at column 0 and current token at +# column 0 are combined as one feature. +patterns = [ + [[-2, 0]], + [[-1, 0]], + [[0, 0]], + [[1, 0]], + [[2, 0]], + [[-1, 0], [0, 0]], + [[0, 0], [1, 0]], + [[-2, 1]], + [[-1, 1]], + [[0, 1]], + [[1, 1]], + [[2, 1]], + [[-2, 1], [-1, 1]], + [[-1, 1], [0, 1]], + [[0, 1], [1, 1]], + [[1, 1], [2, 1]], + [[-2, 1], [-1, 1], [0, 1]], + [[-1, 1], [0, 1], [1, 1]], + [[0, 1], [1, 1], [2, 1]], +] + +dict_label = { + 'B-ADJP': 0, + 'I-ADJP': 1, + 'B-ADVP': 2, + 'I-ADVP': 3, + 'B-CONJP': 4, + 'I-CONJP': 5, + 'B-INTJ': 6, + 'I-INTJ': 7, + 'B-LST': 8, + 'I-LST': 9, + 'B-NP': 10, + 'I-NP': 11, + 'B-PP': 12, + 'I-PP': 13, + 'B-PRT': 14, + 'I-PRT': 15, + 'B-SBAR': 16, + 'I-SBAR': 17, + 'B-UCP': 18, + 'I-UCP': 19, + 'B-VP': 20, + 'I-VP': 21, + 'O': 22 +} + + +def make_features(sequence): + length = len(sequence) + num_features = len(sequence[0]) + + def get_features(pos): + if pos < 0: + return ['#B%s' % -pos] * num_features + if pos >= length: + return ['#E%s' % (pos - length + 1)] * num_features + return sequence[pos] + + for i in xrange(length): + for pattern in patterns: + fname = '/'.join([get_features(i + pos)[f] for pos, f in pattern]) + sequence[i].append(fname) + + +''' +Source file format: +Each line is for one timestep. The features are separated by space. +An empty line indicates end of a sequence. + +cutoff: a list of numbers. If count of a feature is smaller than this, + it will be ignored. +if oov_policy[i] is OOV_POLICY_USE, id 0 is reserved for OOV features of +i-th column. + +return a list of dict for each column +''' + + +def create_dictionaries(filename, cutoff, oov_policy): + def add_to_dict(sequence, dicts): + num_features = len(dicts) + for features in sequence: + l = len(features) + assert l == num_features, "Wrong number of features " + line + for i in xrange(l): + if features[i] in dicts[i]: + dicts[i][features[i]] += 1 + else: + dicts[i][features[i]] = 1 + + num_features = len(cutoff) + dicts = [] + for i in xrange(num_features): + dicts.append(dict()) + + f = gzip.open(filename, 'rb') + + sequence = [] + + for line in f: + line = line.strip() + if not line: + make_features(sequence) + add_to_dict(sequence, dicts) + sequence = [] + continue + features = line.split(' ') + sequence.append(features) + + for i in xrange(num_features): + dct = dicts[i] + n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 + todo = [] + for k, v in dct.iteritems(): + if v < cutoff[i]: + todo.append(k) + else: + dct[k] = n + n += 1 + + if oov_policy[i] == OOV_POLICY_USE: + # placeholder so that len(dct) will be the number of features + # including OOV + dct['#OOV#'] = 0 + + logger.info('column %d dict size=%d, ignored %d' % (i, n, len(todo))) + for k in todo: + del dct[k] + + f.close() + return dicts + + +def initializer(settings, **xargs): + cutoff = [3, 1, 0] + cutoff += [3] * len(patterns) + oov_policy = [OOV_POLICY_IGNORE, OOV_POLICY_ERROR, OOV_POLICY_ERROR] + oov_policy += [OOV_POLICY_IGNORE] * len(patterns) + dicts = create_dictionaries('data/train.txt.gz', cutoff, oov_policy) + dicts[2] = dict_label + settings.dicts = dicts + settings.oov_policy = oov_policy + input_types = [] + num_features = len(dicts) + for i in xrange(num_original_columns): + input_types.append(integer_sequence(len(dicts[i]))) + logger.info("slot %s size=%s" % (i, len(dicts[i]))) + if patterns: + dim = 0 + for i in xrange(num_original_columns, num_features): + dim += len(dicts[i]) + input_types.append(sparse_binary_vector_sequence(dim)) + logger.info("feature size=%s" % dim) + settings.input_types = input_types + + +''' +if oov_policy[i] == OOV_POLICY_USE, features in i-th column which are not +existed in dicts[i] will be assigned to id 0. +if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist +in dicts[i]. +''' + + +@provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) +def process(settings, filename): + input_file = filename + dicts = settings.dicts + oov_policy = settings.oov_policy + + def gen_sample(sequence): + num_features = len(dicts) + sample = [list() for i in xrange(num_original_columns)] + if patterns: + sample.append([]) + for features in sequence: + assert len(features) == num_features, \ + "Wrong number of features: " + line + for i in xrange(num_original_columns): + id = dicts[i].get(features[i], -1) + if id != -1: + sample[i].append(id) + elif oov_policy[i] == OOV_POLICY_IGNORE: + sample[i].append(0xffffffff) + elif oov_policy[i] == OOV_POLICY_ERROR: + logger.fatal("Unknown token: %s" % features[i]) + else: + sample[i].append(0) + + if patterns: + dim = 0 + vec = [] + for i in xrange(num_original_columns, num_features): + id = dicts[i].get(features[i], -1) + if id != -1: + vec.append(dim + id) + elif oov_policy[i] == OOV_POLICY_IGNORE: + pass + elif oov_policy[i] == OOV_POLICY_ERROR: + logger.fatal("Unknown token: %s" % features[i]) + else: + vec.ids.append(dim + 0) + + dim += len(dicts[i]) + sample[-1].append(vec) + return sample + + num_features = len(dicts) + f = gzip.open(input_file, 'rb') + + num_sequences = 0 + sequence = [] + for line in f: + line = line.strip() + if not line: + make_features(sequence) + yield gen_sample(sequence) + sequence = [] + num_sequences += 1 + continue + features = line.split(' ') + sequence.append(features) + + f.close() + + logger.info("num_sequences=%s" % num_sequences) diff --git a/demo/sequence_tagging/linear_crf.py b/demo/sequence_tagging/linear_crf.py new file mode 100644 index 0000000000000..64895742e1b8c --- /dev/null +++ b/demo/sequence_tagging/linear_crf.py @@ -0,0 +1,82 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +import math + +define_py_data_sources2( + train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") + +batch_size = 1 +settings( + learning_method=MomentumOptimizer(), + batch_size=batch_size, + regularization=L2Regularization(batch_size * 1e-4), + average_window=0.5, + learning_rate=1e-1, + learning_rate_decay_a=1e-5, + learning_rate_decay_b=0.25, ) + +num_label_types = 23 + + +def get_simd_size(size): + return int(math.ceil(float(size) / 8)) * 8 + + +# Currently, in order to use sparse_update=True, +# the size has to be aligned. +num_label_types = get_simd_size(num_label_types) + +features = data_layer(name="features", size=76328) +word = data_layer(name="word", size=6778) +pos = data_layer(name="pos", size=44) +chunk = data_layer(name="chunk", size=num_label_types) + +crf_input = fc_layer( + input=features, + size=num_label_types, + act=LinearActivation(), + bias_attr=False, + param_attr=ParamAttr( + initial_std=0, sparse_update=True)) + +crf = crf_layer( + input=crf_input, + label=chunk, + param_attr=ParamAttr( + name="crfw", initial_std=0), ) + +crf_decoding = crf_decoding_layer( + size=num_label_types, + input=crf_input, + label=chunk, + param_attr=ParamAttr(name="crfw"), ) + +sum_evaluator( + name="error", + input=crf_decoding, ) + +chunk_evaluator( + name="chunk_f1", + input=[crf_decoding, chunk], + chunk_scheme="IOB", + num_chunk_types=11, ) + +inputs(word, pos, chunk, features) +outputs(crf) diff --git a/demo/sequence_tagging/readme.md b/demo/sequence_tagging/readme.md new file mode 100644 index 0000000000000..2e17fffb83c53 --- /dev/null +++ b/demo/sequence_tagging/readme.md @@ -0,0 +1,45 @@ +# Sequence Tagging + +This demo is a sequence model for assigning tags to each token in a sentence. The task is described at CONLL2000 Text Chunking task. + +## Download data +```bash +cd demo/sequence_tagging +./data/get_data.sh +``` + +## Train model +```bash +cd demo/sequence_tagging +./train.sh +``` + +## Model description + +We provide two models. One is a linear CRF model (linear_crf.py) with is equivalent to the one at leon.bottou.org/projects/sgd. The second one is a stacked bidirectional RNN and CRF model (rnn_crf.py). +
+ + + + + + + + + + + + + + + + + + + + + + +
Model nameNumber of parametersF1 score
linear_crf 1.8M 0.937
rnn_crf 960K 0.941
+
+
diff --git a/demo/sequence_tagging/rnn_crf.py b/demo/sequence_tagging/rnn_crf.py new file mode 100644 index 0000000000000..90d4bbdddfdb4 --- /dev/null +++ b/demo/sequence_tagging/rnn_crf.py @@ -0,0 +1,120 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +import math + +define_py_data_sources2( + train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") + +batch_size = 16 +settings( + learning_method=MomentumOptimizer(), + batch_size=batch_size, + regularization=L2Regularization(batch_size * 1e-5), + average_window=0.5, + learning_rate=2e-3, + learning_rate_decay_a=5e-7, + learning_rate_decay_b=0.5, ) + +word_dim = 128 +hidden_dim = 128 +with_rnn = True + +initial_std = 1 / math.sqrt(hidden_dim) +param_attr = ParamAttr(initial_std=initial_std) +cpu_layer_attr = ExtraLayerAttribute(device=-1) + +default_device(0) + +num_label_types = 23 + +features = data_layer(name="features", size=76328) +word = data_layer(name="word", size=6778) +pos = data_layer(name="pos", size=44) +chunk = data_layer( + name="chunk", size=num_label_types, layer_attr=cpu_layer_attr) + +emb = embedding_layer( + input=word, size=word_dim, param_attr=ParamAttr(initial_std=0)) + +hidden1 = mixed_layer( + size=hidden_dim, + act=STanhActivation(), + bias_attr=True, + input=[ + full_matrix_projection(emb), table_projection( + pos, param_attr=param_attr) + ]) + +if with_rnn: + rnn1 = recurrent_layer( + act=ReluActivation(), + bias_attr=True, + input=hidden1, + param_attr=ParamAttr(initial_std=0), ) + +hidden2 = mixed_layer( + size=hidden_dim, + act=STanhActivation(), + bias_attr=True, + input=[full_matrix_projection(hidden1)] + + ([full_matrix_projection( + rnn1, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) + +if with_rnn: + rnn2 = recurrent_layer( + reverse=True, + act=ReluActivation(), + bias_attr=True, + input=hidden2, + param_attr=ParamAttr(initial_std=0), ) + +crf_input = mixed_layer( + size=num_label_types, + bias_attr=False, + input=[full_matrix_projection(hidden2), ] + + ([full_matrix_projection( + rnn2, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) + +crf = crf_layer( + input=crf_input, + label=chunk, + param_attr=ParamAttr( + name="crfw", initial_std=0), + layer_attr=cpu_layer_attr, ) + +crf_decoding = crf_decoding_layer( + size=num_label_types, + input=crf_input, + label=chunk, + param_attr=ParamAttr(name="crfw"), + layer_attr=cpu_layer_attr, ) + +sum_evaluator( + name="error", + input=crf_decoding, ) + +chunk_evaluator( + name="chunk_f1", + input=[crf_decoding, chunk], + chunk_scheme="IOB", + num_chunk_types=11, ) + +inputs(word, pos, chunk, features) +outputs(crf) diff --git a/demo/sequence_tagging/train.sh b/demo/sequence_tagging/train.sh new file mode 100755 index 0000000000000..9a706b98d8686 --- /dev/null +++ b/demo/sequence_tagging/train.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +paddle train \ + --config rnn_crf.py \ + --parallel_nn=1 \ + --use_gpu=1 \ + --dot_period=10 \ + --log_period=1000 \ + --test_period=0 \ + --num_passes=10 diff --git a/demo/sequence_tagging/train_linear.sh b/demo/sequence_tagging/train_linear.sh new file mode 100755 index 0000000000000..597b5afea9c63 --- /dev/null +++ b/demo/sequence_tagging/train_linear.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +paddle train \ + --config linear_crf.py \ + --use_gpu=0 \ + --dot_period=100 \ + --log_period=10000 \ + --test_period=0 \ + --num_passes=10 diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index c671f483863c7..b8f26f431eb7a 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -1,10 +1,9 @@ Installing from Sources -================= +========================== * [1. Download and Setup](#download) * [2. Requirements](#requirements) * [3. Build on Ubuntu](#ubuntu) -* [4. Build on Mac OS X](#mac) ## Download and Setup You can download PaddlePaddle from the [github source](https://github.com/gangliao/Paddle). @@ -28,51 +27,26 @@ To compile the source code, your computer must be equipped with GCC >=4.6 or Cla PaddlePaddle supports some build options. To enable it, first you need to install the related libraries. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +
OptionalDescription
WITH_GPUCompile with GPU mode.
WITH_DOUBLECompile with double precision floating-point, default: single precision.
WITH_GLOGCompile with glog. If not found, default: an internal log implementation.
WITH_GFLAGSCompile with gflags. If not found, default: an internal flag implementation.
WITH_TESTINGCompile with gtest for PaddlePaddle's unit testing.
WITH_DOCCompile to generate PaddlePaddle's docs, default: disabled (OFF)
WITH_SWIG_PYCompile with python predict API, default: disabled (OFF).
WITH_STYLE_CHECKCompile with code style check, default: enabled (ON).
+ + + + + + + + + + + + + + + +
OptionalDescription
WITH_GPUCompile with GPU mode.
WITH_DOUBLECompile with double precision floating-point, default: single precision.
WITH_GLOGCompile with glog. If not found, default: an internal log implementation.
WITH_GFLAGSCompile with gflags. If not found, default: an internal flag implementation.
WITH_TESTINGCompile with gtest for PaddlePaddle's unit testing.
WITH_DOC Compile to generate PaddlePaddle's docs, default: disabled (OFF).
WITH_SWIG_PYCompile with python predict API, default: disabled (OFF).
WITH_STYLE_CHECKCompile with code style check, default: enabled (ON).
+ **Note:** - The GPU version works best with Cuda Toolkit 7.5 and cuDNN v5. @@ -178,12 +152,12 @@ As a simple example, consider the following: - **Only CPU** ```bash - cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF + cmake .. -DWITH_GPU=OFF ``` - **GPU** ```bash - cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF + cmake .. -DWITH_GPU=ON ``` - **GPU with doc and swig** @@ -196,7 +170,7 @@ Finally, you can build PaddlePaddle: ```bash # you can add build option here, such as: -cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= +cmake .. -DWITH_GPU=ON -DCMAKE_INSTALL_PREFIX= # please use sudo make install, if you want to install PaddlePaddle into the system make -j `nproc` && make install # set PaddlePaddle installation path in ~/.bashrc @@ -216,122 +190,3 @@ sudo pip install /opt/paddle/share/wheels/*.whl # or just run sudo paddle version ``` - -## Building on Mac OS X - -### Prerequisites -This guide is based on Mac OS X 10.11 (El Capitan). Note that if you are running an up to date version of OS X, -you will already have Python 2.7.10 and Numpy 1.8 installed. - -The best option is to use the package manager homebrew to handle installations and upgrades for you. -To install [homebrew](http://brew.sh/), first open a terminal window (you can find Terminal in the Utilities folder in Applications), and issue the command: - -```bash -# install brew -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -# install pip -easy_install pip -``` - -### Install Dependencies - -- **CPU Dependencies** - - ```bash - # Install fundamental dependents - brew install glog gflags cmake protobuf openblas - - # Install google test on Mac OS X - # Download gtest 1.7.0 - wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz - tar -xvf googletest-release-1.7.0.tar.gz && cd googletest-release-1.7.0 - # Build gtest - mkdir build && cmake .. - make - # Install gtest library - sudo cp -r ../include/gtest /usr/local/include/ - sudo cp lib*.a /usr/local/lib - ``` - -- **GPU Dependencies(optional)** - - To build GPU version, you will need the following installed: - - 1. a CUDA-capable GPU - 2. Mac OS X 10.11 or later - 2. the Clang compiler and toolchain installed using Xcode - 3. NVIDIA CUDA Toolkit (available at http://developer.nvidia.com/cuda-downloads) - 4. NVIDIA cuDNN Library (availabel at https://developer.nvidia.com/cudnn) - - The CUDA development environment relies on tight integration with the host development environment, - including the host compiler and C runtime libraries, and is therefore only supported on - distribution versions that have been qualified for this CUDA Toolkit release. - - 1. After downloading cuDNN library, issue the following commands: - - ```bash - sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local - sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* - ``` - 2. Then you need to set DYLD\_LIBRARY\_PATH, PATH environment variables in ~/.bashrc. - - ```bash - export DYLD_LIBRARY_PATH=/usr/local/cuda/lib:$DYLD_LIBRARY_PATH - export PATH=/usr/local/cuda/bin:$PATH - ``` - -### Build and Install - -As usual, the best option is to create build folder under paddle project directory. - -```bash -mkdir build && cd build -cmake .. -``` - -CMake first check PaddlePaddle's dependencies in system default path. After installing some optional -libraries, corresponding build option will be set automatically (for instance, glog, gtest and gflags). -If still not found, you can manually set it based on CMake error information from your screen. - -As a simple example, consider the following: - -- **Only CPU** - - ```bash - cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF - ``` -- **GPU** - - ```bash - cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF - ``` - -- **GPU with doc and swig** - - ```bash - cmake .. -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON - ``` - -Finally, you can build PaddlePaddle: - -```bash -# you can add build option here, such as: -cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= -# please use sudo make install, if you want to install PaddlePaddle into the system -make -j `nproc` && make install -# set PaddlePaddle installation path in ~/.bashrc -export PATH=/bin:$PATH -``` -**Note:** - -If you set `WITH_SWIG_PY=ON`, related python dependencies also need to be installed. -Otherwise, PaddlePaddle will automatically install python dependencies -at first time when user run paddle commands, such as `paddle version`, `paddle train`. -It may require sudo privileges: - -```bash -# you can run -sudo pip install /opt/paddle/share/wheels/*.whl -# or just run -sudo paddle version -``` \ No newline at end of file diff --git a/doc/build/contribute_to_paddle.md b/doc/build/contribute_to_paddle.md index 06fcff6172075..a9ab69c5f42b8 100644 --- a/doc/build/contribute_to_paddle.md +++ b/doc/build/contribute_to_paddle.md @@ -4,7 +4,7 @@ We sincerely appreciate your contributions. You can use fork and pull request workflow to merge your code. ## Code Requirements -- Your code mush be fully documented by +- Your code must be fully documented by [doxygen](http://www.stack.nl/~dimitri/doxygen/) style. - Make sure the compiler option WITH\_STYLE\_CHECK is on and the compiler passes the code style check. @@ -20,16 +20,30 @@ It's just that simple. ## Clone +Paddle is currently using [git-flow branching model](http://nvie.com/posts/a-successful-git-branching-model/). +The **develop** is the main branch, and other user's branches are feature branches. + Once you've created a fork, you can use your favorite git client to clone your repo or just head straight to the command line: ```shell # Clone your fork to your local machine -git clone https://github.com/USERNAME/Paddle.git +git clone --branch develop https://github.com/USERNAME/Paddle.git +``` +If your repository doesn't contain **develop** branch, just create it by your own. + +```shell +git clone https://github.com/USERNAME/Paddle.git Paddle +cd Paddle +git checkout -b develop # create develop branch. +git remote add upstream https://github.com/baidu/Paddle.git # add upstream to baidu/Paddle +git pull upstream develop # update to upstream ``` + Then you can start to develop by making a local developement branch + ```shell -git checkout -b MY_COOL_STUFF_BRANCH origin/master +git checkout -b MY_COOL_STUFF_BRANCH ``` ## Commit @@ -41,7 +55,7 @@ Commit your changes by following command lines: git status # add modified files git add xx -git commit -m "commit info" +env EDITOR=vim git commit # You can write your comments by vim/nano/emacs. ``` The first line of commit infomation is the title. The second and later lines are the details if any. @@ -63,7 +77,7 @@ git remote -v Update your fork with the latest upstream changes: ```shell -git pull --rebase upstream HEAD +git pull --rebase upstream develop ``` If there are no unique commits locally, git will simply perform a fast-forward. @@ -76,7 +90,7 @@ Now, your local master branch is up-to-date with everything modified upstream. ```shell # push to your repository in Github -git push origin HEAD +git push -u origin MY_COOL_STUFF_BRANCH # create remote branch MY_COOL_STUFF_BRANCH to origin. ``` ## Pull Request @@ -93,9 +107,24 @@ of conflict, you need to do the update manually. You need to do the following on your local repository: ```shell git checkout MY_COOL_STUFF_BRANCH -git pull --rebase upstream HEAD +git pull upstream develop # You may need to resolve the conflict according to the git prompt. # Make and test your code. -git push -f origin HEAD +git push origin MY_COOL_STUFF_BRANCH ``` Now your Pull Request is updated with the latest version. + +## Revise your pull request + +When you revise your pull request according to reviewer's comments, please use 'git commit' instead of 'git commit --amend' to commit your changes so that the reviewers can see the difference between the new pull requrest and the old pull request. + +The possible commands are + +```shell +git checkout MY_COOL_STUFF_BRANCH +git pull upstream develop # update local to newest code base. +# May be some conflicts will occured. +# And develop your cool stuff +env EDITOR=vim git commit # add your revise log +git push origin MY_COOL_STUFF_BRANCH +``` diff --git a/doc/build/docker_install.md b/doc/build/docker_install.md deleted file mode 100644 index 3cd9d1730a22b..0000000000000 --- a/doc/build/docker_install.md +++ /dev/null @@ -1,91 +0,0 @@ -Docker installation guide -==================== -PaddlePaddle provides some pre-compiled binary, including Docker images, ubuntu deb packages. It is welcomed to contributed more installation package of different linux distribution (such as ubuntu, centos, debian, gentoo and so on). We recommend to use Docker images to deploy PaddlePaddle. -## Docker installation - -Docker is a tool designed to make it easier to create, deploy, and run applications by using containers. - -### PaddlePaddle Docker images -There are six Docker images: - -- paddledev/paddle:cpu-latest: PaddlePaddle CPU binary image. -- paddledev/paddle:gpu-latest: PaddlePaddle GPU binary image. -- paddledev/paddle:cpu-devel-latest: PaddlePaddle CPU binary image plus source code. -- paddledev/paddle:gpu-devel-latest: PaddlePaddle GPU binary image plus source code. -- paddledev/paddle:cpu-demo-latest: PaddlePaddle CPU binary image plus source code and demo -- paddledev/paddle:gpu-demo-latest: PaddlePaddle GPU binary image plus source code and demo - -Tags with latest will be replaced by a released version. - -### Download and Run Docker images - -You have to install Docker in your machine which has linux kernel version 3.10+ first. You can refer to the official guide https://docs.docker.com/engine/installation/ for further information. - -You can use ```docker pull ```to download images first, or just launch a container with ```docker run```: -```bash -docker run -it paddledev/paddle:cpu-latest -``` - -If you want to launch container with GPU support, you need to set some environment variables at the same time: - -```bash -export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}" -export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') -docker run -it paddledev/paddle:gpu-latest -``` - -### Notice - -#### Performance - -Since Docker is based on the lightweight virtual containers, the CPU computing performance maintains well. And GPU driver and equipments are all mapped to the container, so the GPU computing performance would not be seriously affected. - -If you use high performance nic, such as RDMA(RoCE 40GbE or IB 56GbE), Ethernet(10GbE), it is recommended to use config "-net = host". - - - - -#### Remote access -If you want to enable ssh access background, you need to build an image by yourself. Please refer to official guide https://docs.docker.com/engine/reference/builder/ for further information. - -Following is a simple Dockerfile with ssh: -```bash -FROM paddledev/paddle - -MAINTAINER PaddlePaddle dev team - -RUN apt-get update -RUN apt-get install -y openssh-server -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd - -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] -``` - -Then you can build an image with Dockerfile and launch a container: - -```bash -# cd into Dockerfile directory -docker build . -t paddle_ssh -# run container, and map host machine port 8022 to container port 22 -docker run -d -p 8022:22 --name paddle_ssh_machine paddle_ssh -``` -Now, you can ssh on port 8022 to access the container, username is root, password is also root: - -```bash -ssh -p 8022 root@YOUR_HOST_MACHINE -``` - - -You can stop and delete the container as following: -```bash -# stop -docker stop paddle_ssh_machine -# delete -docker rm paddle_ssh_machine -``` diff --git a/doc/build/docker_install.rst b/doc/build/docker_install.rst new file mode 100644 index 0000000000000..e95de35f4da35 --- /dev/null +++ b/doc/build/docker_install.rst @@ -0,0 +1,122 @@ +Docker installation guide +========================== + +PaddlePaddle provide the `Docker `_ image. `Docker`_ is a lightweight container utilities. The performance of PaddlePaddle in `Docker`_ container is basically as same as run it in a normal linux. The `Docker`_ is a very convenient way to deliver the binary release for linux programs. + +.. note:: + + The `Docker`_ image is the recommended way to run PaddlePaddle + +PaddlePaddle Docker images +-------------------------- + +There are 12 `images `_ for PaddlePaddle, and the name is :code:`paddle-dev/paddle`, tags are\: + + ++-----------------+------------------+------------------------+-----------------------+ +| | normal | devel | demo | ++=================+==================+========================+=======================+ +| CPU | cpu-latest | cpu-devel-latest | cpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU | gpu-latest | gpu-devel-latest | gpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| CPU WITHOUT AVX | cpu-noavx-latest | cpu-devel-noavx-latest | cpu-demo-noavx-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU WITHOUT AVX | gpu-noavx-latest | gpu-devel-noavx-latest | gpu-demo-noavx-latest | ++-----------------+------------------+------------------------+-----------------------+ + +And the three columns are: + +* normal\: The docker image only contains binary of PaddlePaddle. +* devel\: The docker image contains PaddlePaddle binary, source code and essential build environment. +* demo\: The docker image contains the dependencies to run PaddlePaddle demo. + +And the four rows are: + +* CPU\: CPU Version. Support CPU which has :code:`AVX` instructions. +* GPU\: GPU Version. Support GPU, and cpu has :code:`AVX` instructions. +* CPU WITHOUT AVX\: CPU Version, which support most CPU even doesn't have :code:`AVX` instructions. +* GPU WITHOUT AVX\: GPU Version, which support most CPU even doesn't have :code:`AVX` instructions. + +User can choose any version depends on machine. The following script can help you to detect your CPU support :code:`AVX` or not. + +.. code-block:: bash + + if cat /proc/cpuinfo | grep -q avx ; then echo "Support AVX"; else echo "Not support AVX"; fi + +If the output is :code:`Support AVX`, then you can choose the AVX version of PaddlePaddle, otherwise, you need select :code:`noavx` version of PaddlePaddle. For example, the CPU develop version of PaddlePaddle is :code:`paddle-dev/paddle:cpu-devel-latest`. + +The PaddlePaddle images don't contain any entry command. You need to write your entry command to use this image. See :code:`Remote Access` part or just use following command to run a :code:`bash` + +.. code-block:: bash + + docker run -it paddledev/paddle:cpu-latest /bin/bash + + +Download and Run Docker images +------------------------------ + +You have to install Docker in your machine which has linux kernel version 3.10+ first. You can refer to the official guide https://docs.docker.com/engine/installation/ for further information. + +You can use :code:`docker pull ` to download images first, or just launch a container with :code:`docker run` \: + +.. code-block:: bash + + docker run -it paddledev/paddle:cpu-latest + + +If you want to launch container with GPU support, you need to set some environment variables at the same time: + +.. code-block:: bash + + export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" + export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') + docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:gpu-latest + + +Some notes for docker +--------------------- + +Performance ++++++++++++ + +Since Docker is based on the lightweight virtual containers, the CPU computing performance maintains well. And GPU driver and equipments are all mapped to the container, so the GPU computing performance would not be seriously affected. + +If you use high performance nic, such as RDMA(RoCE 40GbE or IB 56GbE), Ethernet(10GbE), it is recommended to use config "-net = host". + + + + +Remote access ++++++++++++++ + + +If you want to enable ssh access background, you need to build an image by yourself. Please refer to official guide https://docs.docker.com/engine/reference/builder/ for further information. + +Following is a simple Dockerfile with ssh: + +.. literalinclude:: ../../doc_cn/build_and_install/install/paddle_ssh.Dockerfile + +Then you can build an image with Dockerfile and launch a container: + +.. code-block:: bash + + # cd into Dockerfile directory + docker build . -t paddle_ssh + # run container, and map host machine port 8022 to container port 22 + docker run -d -p 8022:22 --name paddle_ssh_machine paddle_ssh + +Now, you can ssh on port 8022 to access the container, username is root, password is also root: + +.. code-block:: bash + + ssh -p 8022 root@YOUR_HOST_MACHINE + +You can stop and delete the container as following: + +.. code-block:: bash + + # stop + docker stop paddle_ssh_machine + # delete + docker rm paddle_ssh_machine diff --git a/doc/build/index.rst b/doc/build/index.rst index d6d0d19e110fc..511cdea145c7f 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -10,31 +10,24 @@ Install PaddlePaddle install_* internal/install_from_jumbo.md + docker_install.rst + ubuntu_install.rst Build from Source ----------------- -If you want to hack and contribute PaddlePaddle source code, following guides can help you\: +.. warning:: -.. toctree:: - :maxdepth: 1 - :glob: + Please use :code:`deb` package or :code:`docker` image to install paddle. The building guide is used for hacking or contributing to PaddlePaddle. + - build_from_source.md - contribute_to_paddle.md - -Docker and Debian Package installation --------------------------------------- - -Note: The installation packages are still in pre-release -state and your experience of installation may not be smooth. +If you want to hack and contribute PaddlePaddle source code, following guides can help you\: -If you want to pack docker image, the following guide can help you\: .. toctree:: :maxdepth: 1 :glob: - docker_install.md - ubuntu_install.md + build_from_source.md + contribute_to_paddle.md diff --git a/doc/build/ubuntu_install.md b/doc/build/ubuntu_install.md deleted file mode 100644 index c30a8f6db5d9e..0000000000000 --- a/doc/build/ubuntu_install.md +++ /dev/null @@ -1,21 +0,0 @@ -Debian Package installation guide -================================= - -## Debian Package installation -Currently , PaddlePaddle only provides ubuntu14.04 debian packages. -There are two versions package, including CPU and GPU. The download address is: - -https://github.com/baidu/Paddle/releases/tag/V0.8.0b0 - - -After downloading PaddlePaddle deb packages, you can run: - -```bash -dpkg -i paddle-0.8.0b-cpu.deb -apt-get install -f -``` -And if you use GPU version deb package, you need to install CUDA toolkit and cuDNN, and set related environment variables(such as LD_LIBRARY_PATH) first. It is normal when `dpkg -i` get errors. `apt-get install -f` will continue install paddle, and install dependences. - -**Note** - -PaddlePaddle package only supports x86 CPU with AVX instructions. If not, you have to download and build from source code. diff --git a/doc/build/ubuntu_install.rst b/doc/build/ubuntu_install.rst new file mode 100644 index 0000000000000..ea8042085bf45 --- /dev/null +++ b/doc/build/ubuntu_install.rst @@ -0,0 +1,25 @@ +Debian Package installation guide +================================= + +PaddlePaddle supports :code:`deb` pacakge. The installation of this :code:`deb` package is tested in ubuntu 14.04, but it should be support other debian based linux, too. + +There are four versions of debian package, :code:`cpu`, :code:`gpu`, :code:`cpu-noavx`, :code:`gpu-noavx`. And :code:`noavx` version is used to support CPU which does not contain :code:`AVX` instructions. The download url of :code:`deb` package is \: https://github.com/baidu/Paddle/releases/ + + +After downloading PaddlePaddle deb packages, you can use :code:`gdebi` install. + +.. code-block:: bash + + gdebi paddle-*.deb + +If :code:`gdebi` is not installed, you can use :code:`sudo apt-get install gdebi` to install it. + +Or you can use following commands to install PaddlePaddle. + +.. code-block:: bash + + dpkg -i paddle-*.deb + apt-get install -f + +And if you use GPU version deb package, you need to install CUDA toolkit and cuDNN, and set related environment variables(such as LD_LIBRARY_PATH) first. It is normal when `dpkg -i` get errors. `apt-get install -f` will continue install paddle, and install dependences. + diff --git a/doc/cluster/opensource/cluster_train.md b/doc/cluster/opensource/cluster_train.md index 4763ede39b049..cb493a88f0318 100644 --- a/doc/cluster/opensource/cluster_train.md +++ b/doc/cluster/opensource/cluster_train.md @@ -1,26 +1,24 @@ -# Cluster Training +# Distributed Training -We provide some simple scripts ```paddle/scripts/cluster_train``` to help you to launch cluster training Job to harness PaddlePaddle's distributed trainning. For MPI and other cluster scheduler refer this naive script to implement more robust cluster training platform by yourself. +In this article, we explain how to run distributed Paddle training jobs on clusters. We will create the distributed version of the single-process training example, [recommendation](https://github.com/baidu/Paddle/tree/develop/demo/recommendation). -The following cluster demo is based on RECOMMENDATION local training demo in PaddlePaddle ```demo/recommendation``` directory. Assuming you enter the ```paddle/scripts/cluster_train/``` directory. +[Scripts](https://github.com/baidu/Paddle/tree/develop/paddle/scripts/cluster_train) used in this article launch distributed jobs via SSH. They also work as a reference for users running more sophisticated cluster management systems like MPI and Kubernetes. -## Pre-requirements +## Prerequisite -Firstly, +1. Aforementioned scripts use a Python library [fabric](http://www.fabfile.org/) to run SSH commands. We can use `pip` to install fabric: -```bash + ```bash pip install fabric -``` - -Secondly, go through installing scripts to install PaddlePaddle at all nodes to make sure demo can run as local mode. For CUDA enabled training, we assume that CUDA is installed in ```/usr/local/cuda```, otherwise missed cuda runtime libraries error could be reported at cluster runtime. In one word, the local training environment should be well prepared for the simple scripts. + ``` -Then you should prepare same ROOT_DIR directory in all nodes. ROOT_DIR is from in cluster_train/conf.py. Assuming that the ROOT_DIR = /home/paddle, you can create ```paddle``` user account as well, at last ```paddle.py``` can ssh connections to all nodes with ```paddle``` user automatically. +1. We need to install PaddlePaddle on all nodes in the cluster. To enable GPUs, we need to install CUDA in `/usr/local/cuda`; otherwise Paddle would report errors at runtime. -At last you can create ssh mutual trust relationship between all nodes for easy ssh login, otherwise ```password``` should be provided at runtime from ```paddle.py```. +1. Set the `ROOT_DIR` variable in [`cluster_train/conf.py`] on all nodes. For convenience, we often create a Unix user `paddle` on all nodes and set `ROOT_DIR=/home/paddle`. In this way, we can write public SSH keys into `/home/paddle/.ssh/authorized_keys` so that user `paddle` can SSH to all nodes without password. ## Prepare Job Workspace -```Job workspace``` is defined as one package directory which contains dependency libraries, train data, test data, model config file and all other related file dependencies. +We refer to the directory where we put dependent libraries, config files, etc., as *workspace*. These ```train/test``` data should be prepared before launching cluster job. To satisfy the requirement that train/test data are placed in different directory from workspace, PADDLE refers train/test data according to index file named as ```train.list/test.list``` which are used in model config file. So the train/test data also contains train.list/test.list two list file. All local training demo already provides scripts to help you create these two files, and all nodes in cluster job will handle files with same logical code in normal condition. diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index ee3fa2a2166f4..659485d9be1b6 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -1,4 +1,4 @@ -# Quick Start Tutorial +# Quick Start This tutorial will teach the basics of deep learning (DL), including how to implement many different models in PaddlePaddle. You will learn how to: - Prepare data into the standardized format that PaddlePaddle accepts. @@ -134,7 +134,7 @@ def process(settings, file_name): You need to add a data provider definition `define_py_data_sources2` in our network configuration. This definition specifies: - The path of the training and testing data (`data/train.list`, `data/test.list`). -- The location of the data provider file (`dataprovider_pow`). +- The location of the data provider file (`dataprovider_bow`). - The function to call to get data. (`process`). - Additional arguments or data. Here it passes the path of word dictionary. @@ -477,7 +477,7 @@ The scripts of data downloading, network configurations, and training scrips are Word embedding 15MB 8.484% -trainer_config.bow.py +trainer_config.emb.py diff --git a/doc/demo/semantic_role_labeling/semantic_role_labeling.md b/doc/demo/semantic_role_labeling/semantic_role_labeling.md index 05fbc8278daf2..890f7314582c6 100644 --- a/doc/demo/semantic_role_labeling/semantic_role_labeling.md +++ b/doc/demo/semantic_role_labeling/semantic_role_labeling.md @@ -1,183 +1,183 @@ -# Semantic Role labeling Tutorial # - -Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]: - - [ A0 He ] [ AM-MOD would ][ AM-NEG n’t ] [ V accept] [ A1 anything of value ] from [A2 those he was writing about ]. - -- V: verb -- A0: acceptor -- A1: thing accepted -- A2: accepted-from -- A3: Attribute -- AM-MOD: modal -- AM-NEG: negation - -Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank. - -To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem. - -## Data Description -The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website. - -To download and process the original data, user just need to execute the following command: - -```bash -cd data -./get_data.sh -``` -Several new files appear in the `data `directory as follows. -```bash -conll05st-release：the test data set of CoNll-2005 shared task -test.wsj.words：the Wall Street Journal data sentences -test.wsj.props: the propositional arguments -src.dict：the dictionary of words in sentences -tgt.dict：the labels dictionary -feature: the extracted features from data set -``` - -## Training -### DB-LSTM -Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit. - -Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model. - -The following figure shows a temporal expanded 2-layer DB-LSTM network. -
-![pic](./network_arch.png) -
- -### Features -Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark mr = 1 to denote the argument position if it locates in the predicate context region, or mr = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]: -
-![pic](./feature.jpg) -
- -In this sample, the coresponding labelled sentence is: - -[ A1 A record date ] has [ AM-NEG n't ] been [ V set ] . - -In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`. - -### Data Provider - -`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots. -``` -def hook(settings, word_dict, label_dict, **kwargs): - settings.word_dict = word_dict - settings.label_dict = label_dict - #all inputs are integral and sequential type - settings.slots = [ - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(2), - integer_value_sequence(len(label_dict))] -``` -The corresponding data iterator is as following: -``` -@provider(use_seq=True, init_hook=hook) -def process(obj, file_name): - with open(file_name, 'r') as fdata: - for line in fdata: - sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = line.strip().split('\t') - words = sentence.split() - sen_len = len(words) - word_slot = [obj.word_dict.get(w, UNK_IDX) for w in words] - - predicate_slot = [obj.word_dict.get(predicate, UNK_IDX)] * sen_len - ctx_n1_slot = [obj.word_dict.get(ctx_n1, UNK_IDX) ] * sen_len - ctx_0_slot = [obj.word_dict.get(ctx_0, UNK_IDX) ] * sen_len - ctx_p1_slot = [obj.word_dict.get(ctx_p1, UNK_IDX) ] * sen_len - - marks = mark.split() - mark_slot = [int(w) for w in marks] - - label_list = label.split() - label_slot = [obj.label_dict.get(w) for w in label_list] - - yield word_slot, predicate_slot, ctx_n1_slot, ctx_0_slot, ctx_p1_slot, mark_slot, label_slot -``` -The `process`function yield 7 lists which are six features and labels. - -### Neural Network Config -`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure. - -Seven `data_layer` load instances from data provider. Six features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels. - -### Run Training -The script for training is `train.sh`, user just need to execute: -```bash - ./train.sh -``` -The content in `train.sh`: -``` -paddle train \ - --config=./db_lstm.py \ - --save_dir=./output \ - --trainer_count=4 \ - --log_period=10 \ - --num_passes=500 \ - --use_gpu=false \ - --show_parameter_stats_period=10 \ - --test_all_data_in_one_period=1 \ -2>&1 | tee 'train.log' -``` - -- \--config=./db_lstm.py : network config file. -- \--save_di=./output: output path to save models. -- \--trainer_count=4 : set thread number (or GPU count). -- \--log_period=10 : print log every 20 batches. -- \--num_passes=500: set pass number, one pass in PaddlePaddle means training all samples in dataset one time. -- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train. -- \--show_parameter_stats_period=10: show parameter statistic every 100 batches. -- \--test_all_data_in_one_period=1: test all data in every testing. - - -After training, the models will be saved in directory `output`. - -### Run testing -The script for testing is `test.sh`, user just need to execute: -```bash - ./test.sh -``` -The main part in `tesh.sh` -``` -paddle train \ - --config=./db_lstm.py \ - --model_list=$model_list \ - --job=test \ - --config_args=is_test=1 \ -``` - - - \--config=./db_lstm.py: network config file - - \--model_list=$model_list.list: model list file - - \--job=test: indicate the test job - - \--config_args=is_test=1: flag to indicate test - - -### Run prediction -The script for prediction is `predict.sh`, user just need to execute: -```bash - ./predict.sh - -``` -In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file -``` -python predict.py - -c $config_file - -w $model_path - -l $label_file - -d $dict_file - -i $input_file -``` - -`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix. - -After prediction, the result is saved in `predict.res`. - -## Reference -[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005. - -[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015. +# Semantic Role labeling Tutorial # + +Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]: + + [ A0 He ] [ AM-MOD would ][ AM-NEG n’t ] [ V accept] [ A1 anything of value ] from [A2 those he was writing about ]. + +- V: verb +- A0: acceptor +- A1: thing accepted +- A2: accepted-from +- A3: Attribute +- AM-MOD: modal +- AM-NEG: negation + +Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank. + +To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem. + +## Data Description +The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website. + +To download and process the original data, user just need to execute the following command: + +```bash +cd data +./get_data.sh +``` +Several new files appear in the `data `directory as follows. +```bash +conll05st-release：the test data set of CoNll-2005 shared task +test.wsj.words：the Wall Street Journal data sentences +test.wsj.props: the propositional arguments +src.dict：the dictionary of words in sentences +tgt.dict：the labels dictionary +feature: the extracted features from data set +``` + +## Training +### DB-LSTM +Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit. + +Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model. + +The following figure shows a temporal expanded 2-layer DB-LSTM network. +
+![pic](./network_arch.png) +
+ +### Features +Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark mr = 1 to denote the argument position if it locates in the predicate context region, or mr = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]: +
+![pic](./feature.jpg) +
+ +In this sample, the coresponding labelled sentence is: + +[ A1 A record date ] has [ AM-NEG n't ] been [ V set ] . + +In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`. + +### Data Provider + +`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots. +``` +def hook(settings, word_dict, label_dict, **kwargs): + settings.word_dict = word_dict + settings.label_dict = label_dict + #all inputs are integral and sequential type + settings.slots = [ + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(2), + integer_value_sequence(len(label_dict))] +``` +The corresponding data iterator is as following: +``` +@provider(use_seq=True, init_hook=hook) +def process(obj, file_name): + with open(file_name, 'r') as fdata: + for line in fdata: + sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = line.strip().split('\t') + words = sentence.split() + sen_len = len(words) + word_slot = [obj.word_dict.get(w, UNK_IDX) for w in words] + + predicate_slot = [obj.word_dict.get(predicate, UNK_IDX)] * sen_len + ctx_n1_slot = [obj.word_dict.get(ctx_n1, UNK_IDX) ] * sen_len + ctx_0_slot = [obj.word_dict.get(ctx_0, UNK_IDX) ] * sen_len + ctx_p1_slot = [obj.word_dict.get(ctx_p1, UNK_IDX) ] * sen_len + + marks = mark.split() + mark_slot = [int(w) for w in marks] + + label_list = label.split() + label_slot = [obj.label_dict.get(w) for w in label_list] + + yield word_slot, predicate_slot, ctx_n1_slot, ctx_0_slot, ctx_p1_slot, mark_slot, label_slot +``` +The `process`function yield 7 lists which are six features and labels. + +### Neural Network Config +`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure. + +Seven `data_layer` load instances from data provider. Six features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels. + +### Run Training +The script for training is `train.sh`, user just need to execute: +```bash + ./train.sh +``` +The content in `train.sh`: +``` +paddle train \ + --config=./db_lstm.py \ + --save_dir=./output \ + --trainer_count=4 \ + --log_period=10 \ + --num_passes=500 \ + --use_gpu=false \ + --show_parameter_stats_period=10 \ + --test_all_data_in_one_period=1 \ +2>&1 | tee 'train.log' +``` + +- \--config=./db_lstm.py : network config file. +- \--save_di=./output: output path to save models. +- \--trainer_count=4 : set thread number (or GPU count). +- \--log_period=10 : print log every 20 batches. +- \--num_passes=500: set pass number, one pass in PaddlePaddle means training all samples in dataset one time. +- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train. +- \--show_parameter_stats_period=10: show parameter statistic every 100 batches. +- \--test_all_data_in_one_period=1: test all data in every testing. + + +After training, the models will be saved in directory `output`. + +### Run testing +The script for testing is `test.sh`, user just need to execute: +```bash + ./test.sh +``` +The main part in `tesh.sh` +``` +paddle train \ + --config=./db_lstm.py \ + --model_list=$model_list \ + --job=test \ + --config_args=is_test=1 \ +``` + + - \--config=./db_lstm.py: network config file + - \--model_list=$model_list.list: model list file + - \--job=test: indicate the test job + - \--config_args=is_test=1: flag to indicate test + + +### Run prediction +The script for prediction is `predict.sh`, user just need to execute: +```bash + ./predict.sh + +``` +In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file +``` +python predict.py + -c $config_file + -w $model_path + -l $label_file + -d $dict_file + -i $input_file +``` + +`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix. + +After prediction, the result is saved in `predict.res`. + +## Reference +[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005. + +[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015. diff --git a/doc/index.md b/doc/index.md index df03a33fac98c..cbd08ba52abe5 100644 --- a/doc/index.md +++ b/doc/index.md @@ -3,11 +3,12 @@ PaddlePaddle Documentation User Guide ---------- +* [Introduction](introduction/index.md) * [Quick Start](demo/quick_start/index_en.md) * [Build and Installation](build/index.rst) * [Contribute Code](build/contribute_to_paddle.md) * [User Interface](ui/index.md) -* [Model Config Interface](ui/api/trainer_config_helpers/index.md) +* [Model Config Interface](ui/api/trainer_config_helpers/index.rst) * [Example and Demo](demo/index.md) * [Cluster Train](cluster/index.md) diff --git a/doc/introduction/index.md b/doc/introduction/index.md new file mode 100644 index 0000000000000..01f52031a1d02 --- /dev/null +++ b/doc/introduction/index.md @@ -0,0 +1,100 @@ +# Introduction + +PaddlePaddle is a deep learning platform open-sourced by Baidu. With PaddlePaddle, you can easily train a classic neural network within a couple lines of configuration, or you can build sophisticated models that provide state-of-the-art performance on difficult learning tasks like sentiment analysis, machine translation, image caption and so on. + +## 1. A Classic Problem + +Now, to give you a hint of what using PaddlePaddle looks like, let's start with a fundamental learning problem - **simple linear regression** : you have observed a set of two-dimensional data points of `X` and `Y`, where `X` is an explanatory variable and `Y` is corresponding dependent variable, and you want to recover the underlying correlation between `X` and `Y`. Linear regression can be used in many practical scenarios. For example, `X` can be a variable about house size, and `Y` a variable about house price. You can build a model that captures relationship between them by observing real estate markets. + +## 2. Prepare the Data + +Suppose the true relationship can be characterized as `Y = 2X + 0.3`, let's see how to recover this pattern only from observed data. Here is a piece of python code that feeds synthetic data to PaddlePaddle. The code is pretty self-explanatory, the only extra thing you need to add for PaddlePaddle is a definition of input data types. + +```python +# dataprovider.py +from paddle.trainer.PyDataProvider2 import * +import random + +# define data types of input: 2 real numbers +@provider(input_types=[dense_vector(1), dense_vector(1)],use_seq=False) +def process(settings, input_file): + for i in xrange(2000): + x = random.random() + yield [x], [2*x+0.3] +``` + +## 3. Train a NeuralNetwork in PaddlePaddle + +To recover this relationship between `X` and `Y`, we use a neural network with one layer of linear activation units and a square error cost layer. Don't worry if you are not familiar with these terminologies, it's just saying that we are starting from a random line `Y' = wX + b` , then we gradually adapt `w` and `b` to minimize the difference between `Y'` and `Y`. Here is what it looks like in PaddlePaddle: + +```python +# trainer_config.py +from paddle.trainer_config_helpers import * + +# 1. read data. Suppose you saved above python code as dataprovider.py +data_file = 'empty.list' +with open(data_file, 'w') as f: f.writelines(' ') +define_py_data_sources2(train_list=data_file, test_list=None, + module='dataprovider', obj='process',args={}) + +# 2. learning algorithm +settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) + +# 3. Network configuration +x = data_layer(name='x', size=1) +y = data_layer(name='y', size=1) +y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) +cost = regression_cost(input=y_predict, label=y) +outputs(cost) +``` + +Some of the most fundamental usages of PaddlePaddle are demonstrated: + +- The first part shows how to feed data into PaddlePaddle. In general cases, PaddlePaddle reads raw data from a list of files, and then do some user-defined process to get real input. In this case, we only need to create a placeholder file since we are generating synthetic data on the fly. + +- The second part describes learning algorithm. It defines in what ways adjustments are made to model parameters. PaddlePaddle provides a rich set of optimizers, but a simple momentum based optimizer will suffice here, and it processes 12 data points each time. + +- Finally, the network configuration. It usually is as simple as "stacking" layers. Three kinds of layers are used in this configuration: + - **Data Layer**: a network always starts with one or more data layers. They provide input data to the rest of the network. In this problem, two data layers are used respectively for `X` and `Y`. + - **FC Layer**: FC layer is short for Fully Connected Layer, which connects all the input units to current layer and does the actual computation specified as activation function. Computation layers like this are the fundamental building blocks of a deeper model. + - **Cost Layer**: in training phase, cost layers are usually the last layers of the network. They measure the performance of current model, and provide guidence to adjust parameters. + +Now that everything is ready, you can train the network with a simple command line call: + ``` + paddle train --config=trainer_config.py --save_dir=./output --num_passes=30 + ``` + +This means that PaddlePaddle will train this network on the synthectic dataset for 30 passes, and save all the models under path `./output`. You will see from the messages printed out during training phase that the model cost is decreasing as time goes by, which indicates we are getting a closer guess. + + +## 4. Evaluate the Model + +Usually, a different dataset that left out during training phase should be used to evalute the models. However, we are lucky enough to know the real answer: `w=2, b=0.3`, thus a better option is to check out model parameters directly. + +In PaddlePaddle, training is just to get a collection of model parameters, which are `w` and `b` in this case. Each parameter is saved in an individual file in the popular `numpy` array format. Here is the code that reads parameters from last pass. + +```python +import numpy as np +import os + +def load(file_name): + with open(file_name, 'rb') as f: + f.read(16) # skip header for float type. + return np.fromfile(f, dtype=np.float32) + +print 'w=%.6f, b=%.6f' % (load('output/pass-00029/w'), load('output/pass-00029/b')) +# w=1.999743, b=0.300137 +``` + +
![](./parameters.png)
+ +Although starts from a random guess, you can see that value of `w` changes quickly towards 2 and `b` changes quickly towards 0.3. In the end, the predicted line is almost identical with real answer. + +There, you have recovered the underlying pattern between `X` and `Y` only from observed data. + + +## 5. Where to Go from Here + +- Build and Installation +- Quick Start +- Example and Demo diff --git a/doc/introduction/parameters.png b/doc/introduction/parameters.png new file mode 120000 index 0000000000000..f47e74c94fffa --- /dev/null +++ b/doc/introduction/parameters.png @@ -0,0 +1 @@ +../../doc_cn/introduction/parameters.png \ No newline at end of file diff --git a/doc/source/gserver/layers/layer.rst b/doc/source/gserver/layers/layer.rst index 807b22ca140ee..4b8e149505f06 100644 --- a/doc/source/gserver/layers/layer.rst +++ b/doc/source/gserver/layers/layer.rst @@ -465,6 +465,11 @@ SumOfSquaresCostLayer .. doxygenclass:: paddle::SumOfSquaresCostLayer :members: +SumCostLayer +````````````````````` +.. doxygenclass:: paddle::SumCostLayer + :members: + CosSimLayer ----------- .. doxygenclass:: paddle::CosSimLayer diff --git a/doc/ui/api/trainer_config_helpers/activations.rst b/doc/ui/api/trainer_config_helpers/activations.rst index c4e14ed779efb..269e6491e7ebe 100644 --- a/doc/ui/api/trainer_config_helpers/activations.rst +++ b/doc/ui/api/trainer_config_helpers/activations.rst @@ -1,3 +1,7 @@ +=========== +Activations +=========== + BaseActivation ============== @@ -32,6 +36,13 @@ LinearActivation .. automodule:: paddle.trainer_config_helpers.activations :members: LinearActivation :noindex: + +LogActivation +================== + +.. automodule:: paddle.trainer_config_helpers.activations + :members: LogActivation + :noindex: SquareActivation ================ @@ -95,4 +106,3 @@ STanhActivation .. automodule:: paddle.trainer_config_helpers.activations :members: STanhActivation :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/activations_index.rst b/doc/ui/api/trainer_config_helpers/activations_index.rst deleted file mode 100644 index 1c0b71ab77eec..0000000000000 --- a/doc/ui/api/trainer_config_helpers/activations_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Activations -=========== - -.. toctree:: - :maxdepth: 3 - - activations.rst diff --git a/doc/ui/api/trainer_config_helpers/evaluators.rst b/doc/ui/api/trainer_config_helpers/evaluators.rst index 0586c9907e472..d6a79c13e2316 100644 --- a/doc/ui/api/trainer_config_helpers/evaluators.rst +++ b/doc/ui/api/trainer_config_helpers/evaluators.rst @@ -1,3 +1,7 @@ +========== +Evaluators +========== + Base ==== .. automodule:: paddle.trainer_config_helpers.evaluators diff --git a/doc/ui/api/trainer_config_helpers/evaluators_index.rst b/doc/ui/api/trainer_config_helpers/evaluators_index.rst deleted file mode 100644 index 298de3e1a32d3..0000000000000 --- a/doc/ui/api/trainer_config_helpers/evaluators_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Evaluators -========== - -.. toctree:: - :maxdepth: 3 - - evaluators.rst diff --git a/doc/ui/api/trainer_config_helpers/index.md b/doc/ui/api/trainer_config_helpers/index.md deleted file mode 100644 index 00fa99bb3fa4c..0000000000000 --- a/doc/ui/api/trainer_config_helpers/index.md +++ /dev/null @@ -1,10 +0,0 @@ -# Model Config Interface - -* [Optimizer](optimizers_index.rst) -* [Data Source](data_sources.rst) -* [Layers](layers_index.rst) -* [Activations](activations_index.rst) -* [Poolings](poolings_index.rst) -* [Networks](networks_index.rst) -* [Evaluators](evaluators_index.rst) -* [Parameter and Extra Layer Attribute](attrs.rst) diff --git a/doc/ui/api/trainer_config_helpers/index.rst b/doc/ui/api/trainer_config_helpers/index.rst new file mode 100644 index 0000000000000..8395eb75710b3 --- /dev/null +++ b/doc/ui/api/trainer_config_helpers/index.rst @@ -0,0 +1,14 @@ +Model Config Interface +====================== + +.. toctree:: + :maxdepth: 1 + + optimizers.rst + data_sources.rst + layers.rst + activations.rst + poolings.rst + networks.rst + evaluators.rst + attrs.rst diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index c1d7a7ce81530..4a02af3969932 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -1,3 +1,7 @@ +====== +Layers +====== + Base ====== @@ -46,6 +50,12 @@ conv_operator :members: conv_operator :noindex: +conv_projection +--------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: conv_projection + :noindex: + conv_shift_layer ------------------ .. automodule:: paddle.trainer_config_helpers.layers @@ -71,6 +81,18 @@ img_pool_layer -------------- .. automodule:: paddle.trainer_config_helpers.layers :members: img_pool_layer + :noindex: + +spp_layer +-------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: spp_layer + :noindex: + +maxout_layer +------------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: maxout_layer :noindex: Norm Layer @@ -130,6 +152,12 @@ gru_step_layer Recurrent Layer Group ===================== +memory +------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: memory + :noindex: + recurrent_group --------------- .. automodule:: paddle.trainer_config_helpers.layers @@ -163,6 +191,12 @@ embedding_layer :members: embedding_layer :noindex: +scaling_projection +----------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: scaling_projection + :noindex: + dotmul_projection ----------------- .. automodule:: paddle.trainer_config_helpers.layers @@ -242,6 +276,12 @@ expand_layer :members: expand_layer :noindex: +repeat_layer +------------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: repeat_layer + :noindex: + Math Layers =========== @@ -263,6 +303,12 @@ interpolation_layer :members: interpolation_layer :noindex: +bilinear_interp_layer +---------------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: bilinear_interp_layer + :noindex: + power_layer ----------- .. automodule:: paddle.trainer_config_helpers.layers @@ -371,12 +417,24 @@ ctc_layer :members: ctc_layer :noindex: +nce_layer +----------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: nce_layer + :noindex: + hsigmoid --------- .. automodule:: paddle.trainer_config_helpers.layers :members: hsigmoid :noindex: +sum_cost +--------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: sum_cost + :noindex: + Check Layer ============ diff --git a/doc/ui/api/trainer_config_helpers/layers_index.rst b/doc/ui/api/trainer_config_helpers/layers_index.rst deleted file mode 100644 index c0daab152148c..0000000000000 --- a/doc/ui/api/trainer_config_helpers/layers_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Layers -====== - -.. toctree:: - :maxdepth: 3 - - layers.rst diff --git a/doc/ui/api/trainer_config_helpers/networks.rst b/doc/ui/api/trainer_config_helpers/networks.rst index 2a15b34eaea0b..29c52c5ce3078 100644 --- a/doc/ui/api/trainer_config_helpers/networks.rst +++ b/doc/ui/api/trainer_config_helpers/networks.rst @@ -1,3 +1,9 @@ +======== +Networks +======== + +The networks module contains pieces of neural network that combine multiple layers. + NLP === @@ -111,4 +117,3 @@ outputs .. automodule:: paddle.trainer_config_helpers.networks :members: outputs :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/networks_index.rst b/doc/ui/api/trainer_config_helpers/networks_index.rst deleted file mode 100644 index 17bc4dfaa6c4e..0000000000000 --- a/doc/ui/api/trainer_config_helpers/networks_index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Networks -======== - -The networks module contains pieces of neural network that combine multiple layers. - -.. toctree:: - :maxdepth: 3 - - networks.rst diff --git a/doc/ui/api/trainer_config_helpers/optimizers.rst b/doc/ui/api/trainer_config_helpers/optimizers.rst index b487fec64c4eb..7ca4e34156e27 100644 --- a/doc/ui/api/trainer_config_helpers/optimizers.rst +++ b/doc/ui/api/trainer_config_helpers/optimizers.rst @@ -1,3 +1,7 @@ +========== +Optimizers +========== + BaseSGDOptimizer ================ .. automodule:: paddle.trainer_config_helpers.optimizers @@ -51,4 +55,3 @@ settings .. automodule:: paddle.trainer_config_helpers.optimizers :members: settings :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/optimizers_index.rst b/doc/ui/api/trainer_config_helpers/optimizers_index.rst deleted file mode 100644 index f39f94f0cd6e1..0000000000000 --- a/doc/ui/api/trainer_config_helpers/optimizers_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Optimizers -========== - -.. toctree:: - :maxdepth: 3 - - optimizers.rst diff --git a/doc/ui/api/trainer_config_helpers/poolings.rst b/doc/ui/api/trainer_config_helpers/poolings.rst index caadec639383a..66566809d26f5 100644 --- a/doc/ui/api/trainer_config_helpers/poolings.rst +++ b/doc/ui/api/trainer_config_helpers/poolings.rst @@ -1,3 +1,7 @@ +======== +Poolings +======== + BasePoolingType =============== .. automodule:: paddle.trainer_config_helpers.poolings @@ -27,4 +31,3 @@ SquareRootNPooling .. automodule:: paddle.trainer_config_helpers.poolings :members: SquareRootNPooling :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/poolings_index.rst b/doc/ui/api/trainer_config_helpers/poolings_index.rst deleted file mode 100644 index 250d3fa69c0dc..0000000000000 --- a/doc/ui/api/trainer_config_helpers/poolings_index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Poolings -======== - -These pooling types are used for sequence input, not for images. - -.. toctree:: - :maxdepth: 3 - - poolings.rst diff --git a/doc/ui/cmd_argument/argument_outline.md b/doc/ui/cmd_argument/argument_outline.md index 98dadc270dcac..d6cc2c6ed7cc1 100644 --- a/doc/ui/cmd_argument/argument_outline.md +++ b/doc/ui/cmd_argument/argument_outline.md @@ -183,7 +183,7 @@ It looks like there are a lot of arguments. However, most of them are for develo -GPUgpu_id +GPUgpu_id √√√√ @@ -207,6 +207,11 @@ It looks like there are a lot of arguments. However, most of them are for develo √√√√ + +cudnn_conv_workspace_limit_in_mb +√√√√ + + RNN beam_size diff --git a/doc/ui/cmd_argument/detail_introduction.md b/doc/ui/cmd_argument/detail_introduction.md index 0d0362d022a72..07608e5edf740 100644 --- a/doc/ui/cmd_argument/detail_introduction.md +++ b/doc/ui/cmd_argument/detail_introduction.md @@ -163,6 +163,10 @@ - Choose path to dynamic load NVIDIA CUDA library, for instance, /usr/local/cuda/lib64. [Default]: LD_LIBRARY_PATH - type: string (default: "", null) +* `--cudnn_conv_workspace_limit_in_mb` + - Specify cuDNN max workspace limit, in units MB, 4096MB=4GB by default. + - type: int32 (default: 4096MB=4GB) + ## NLP: RNN/LSTM/GRU * `--rnn_use_batch` - Whether to use batch method for calculation in simple RecurrentLayer. diff --git a/doc/ui/predict/predict_sample.py b/doc/ui/predict/predict_sample.py index d55d2c730dece..63e8b36d26057 100644 --- a/doc/ui/predict/predict_sample.py +++ b/doc/ui/predict/predict_sample.py @@ -16,82 +16,113 @@ from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -TEST_DATA = [[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.215686, - 0.533333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.67451, - 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.886275, - 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.192157, 0.070588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.670588, 0.992157, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.117647, 0.933333, 0.858824, 0.313725, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0.090196, 0.858824, 0.992157, 0.831373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.141176, - 0.992157, 0.992157, 0.611765, 0.054902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.258824, 0.992157, 0.992157, - 0.529412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.368627, 0.992157, 0.992157, 0.419608, 0.003922, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0.094118, 0.835294, 0.992157, 0.992157, 0.517647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.603922, 0.992157, - 0.992157, 0.992157, 0.603922, 0.545098, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0.447059, 0.992157, 0.992157, - 0.956863, 0.062745, 0, 0, 0, 0, 0, 0, 0, 0, 0.011765, 0.666667, 0.992157, 0.992157, 0.992157, 0.992157, - 0.992157, 0.745098, 0.137255, 0, 0, 0, 0, 0, 0.152941, 0.866667, 0.992157, 0.992157, 0.521569, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, 0.992157, 0.992157, 0.803922, 0.352941, 0.745098, 0.992157, - 0.945098, 0.317647, 0, 0, 0, 0, 0.580392, 0.992157, 0.992157, 0.764706, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0.070588, 0.992157, 0.992157, 0.776471, 0.043137, 0, 0.007843, 0.27451, 0.882353, 0.941176, 0.176471, - 0, 0, 0.180392, 0.898039, 0.992157, 0.992157, 0.313725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, - 0.992157, 0.713725, 0, 0, 0, 0, 0.627451, 0.992157, 0.729412, 0.062745, 0, 0.509804, 0.992157, 0.992157, - 0.776471, 0.035294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.494118, 0.992157, 0.992157, 0.968627, 0.168627, 0, 0, - 0, 0.423529, 0.992157, 0.992157, 0.364706, 0, 0.717647, 0.992157, 0.992157, 0.317647, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0.533333, 0.992157, 0.984314, 0.945098, 0.603922, 0, 0, 0, 0.003922, 0.466667, 0.992157, - 0.988235, 0.976471, 0.992157, 0.992157, 0.788235, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.686275, - 0.882353, 0.364706, 0, 0, 0, 0, 0, 0, 0.098039, 0.588235, 0.992157, 0.992157, 0.992157, 0.980392, - 0.305882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.101961, 0.67451, 0.321569, 0, 0, 0, 0, 0, 0, 0, 0.105882, - 0.733333, 0.976471, 0.811765, 0.713725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.65098, 0.992157, - 0.321569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.25098, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0.94902, 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.968627, - 0.764706, 0.152941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.498039, - 0.25098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.298039, 0.333333, 0.333333, 0.333333, 0.337255, 0.333333, - 0.333333, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.027451, 0.223529, 0.776471, - 0.964706, 0.988235, 0.988235, 0.988235, 0.992157, 0.988235, 0.988235, 0.780392, 0.098039, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.14902, 0.698039, 0.988235, 0.992157, 0.988235, 0.901961, 0.87451, - 0.568627, 0.882353, 0.976471, 0.988235, 0.988235, 0.501961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.188235, 0.647059, 0.988235, 0.988235, 0.745098, 0.439216, 0.098039, 0, 0, 0, 0.572549, 0.988235, - 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, 0.933333, 0.992157, 0.941176, - 0.247059, 0, 0, 0, 0, 0, 0, 0.188235, 0.898039, 0.992157, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.039216, 0.639216, 0.933333, 0.988235, 0.913725, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0.113725, 0.843137, - 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.235294, 0.988235, 0.992157, 0.988235, 0.815686, - 0.07451, 0, 0, 0, 0, 0, 0, 0, 0.333333, 0.988235, 0.988235, 0.552941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.211765, 0.878431, 0.988235, 0.992157, 0.701961, 0.329412, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0.698039, - 0.988235, 0.913725, 0.145098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.890196, 0.988235, 0.988235, - 0.745098, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.882353, 0.988235, 0.568627, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0.2, 0.933333, 0.992157, 0.992157, 0.992157, 0.447059, 0.294118, 0, 0, 0, 0, 0, 0, 0, 0, 0.447059, - 0.992157, 0.768627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.988235, 0.988235, 0.988235, - 0.992157, 0.47451, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.933333, 0.87451, 0.509804, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0.992157, 0.988235, 0.937255, 0.792157, 0.988235, 0.894118, 0.082353, 0, 0, 0, 0, 0, 0, - 0.027451, 0.647059, 0.992157, 0.654902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.913725, - 0.329412, 0.376471, 0.184314, 0, 0, 0, 0, 0, 0, 0.027451, 0.513725, 0.988235, 0.635294, 0.219608, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.929412, 0.988235, 0.988235, 0.741176, 0.309804, 0, 0, 0, 0, - 0, 0, 0.529412, 0.988235, 0.678431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.223529, 0.992157, - 0.992157, 1, 0.992157, 0.992157, 0.992157, 0.992157, 1, 0.992157, 0.992157, 0.882353, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.023529, 0.478431, 0.654902, 0.658824, 0.952941, 0.988235, 0.988235, - 0.988235, 0.992157, 0.988235, 0.729412, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0.196078, 0.647059, 0.764706, 0.764706, 0.768627, 0.580392, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0]]] +TEST_DATA = [[[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.215686, 0.533333, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.67451, 0.992157, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.070588, 0.886275, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.192157, + 0.070588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.670588, 0.992157, + 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.117647, 0.933333, 0.858824, 0.313725, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.090196, 0.858824, 0.992157, 0.831373, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.141176, 0.992157, 0.992157, 0.611765, 0.054902, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.258824, 0.992157, 0.992157, 0.529412, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.368627, 0.992157, 0.992157, 0.419608, 0.003922, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.094118, 0.835294, 0.992157, 0.992157, 0.517647, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.603922, 0.992157, 0.992157, 0.992157, 0.603922, + 0.545098, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0.447059, 0.992157, 0.992157, + 0.956863, 0.062745, 0, 0, 0, 0, 0, 0, 0, 0, 0.011765, 0.666667, 0.992157, + 0.992157, 0.992157, 0.992157, 0.992157, 0.745098, 0.137255, 0, 0, 0, 0, 0, + 0.152941, 0.866667, 0.992157, 0.992157, 0.521569, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.070588, 0.992157, 0.992157, 0.992157, 0.803922, 0.352941, 0.745098, + 0.992157, 0.945098, 0.317647, 0, 0, 0, 0, 0.580392, 0.992157, 0.992157, + 0.764706, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, 0.992157, + 0.776471, 0.043137, 0, 0.007843, 0.27451, 0.882353, 0.941176, 0.176471, 0, + 0, 0.180392, 0.898039, 0.992157, 0.992157, 0.313725, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0.070588, 0.992157, 0.992157, 0.713725, 0, 0, 0, 0, 0.627451, + 0.992157, 0.729412, 0.062745, 0, 0.509804, 0.992157, 0.992157, 0.776471, + 0.035294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.494118, 0.992157, 0.992157, + 0.968627, 0.168627, 0, 0, 0, 0.423529, 0.992157, 0.992157, 0.364706, 0, + 0.717647, 0.992157, 0.992157, 0.317647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.533333, 0.992157, 0.984314, 0.945098, 0.603922, 0, 0, 0, 0.003922, + 0.466667, 0.992157, 0.988235, 0.976471, 0.992157, 0.992157, 0.788235, + 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.686275, 0.882353, 0.364706, 0, + 0, 0, 0, 0, 0, 0.098039, 0.588235, 0.992157, 0.992157, 0.992157, 0.980392, + 0.305882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.101961, 0.67451, 0.321569, + 0, 0, 0, 0, 0, 0, 0, 0.105882, 0.733333, 0.976471, 0.811765, 0.713725, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.65098, 0.992157, 0.321569, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0.25098, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 0.94902, 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.968627, 0.764706, 0.152941, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.498039, 0.25098, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 +]], [[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.298039, 0.333333, 0.333333, 0.333333, 0.337255, + 0.333333, 0.333333, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0.027451, 0.223529, 0.776471, 0.964706, 0.988235, 0.988235, 0.988235, + 0.992157, 0.988235, 0.988235, 0.780392, 0.098039, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.14902, 0.698039, 0.988235, 0.992157, 0.988235, 0.901961, + 0.87451, 0.568627, 0.882353, 0.976471, 0.988235, 0.988235, 0.501961, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.647059, 0.988235, 0.988235, + 0.745098, 0.439216, 0.098039, 0, 0, 0, 0.572549, 0.988235, 0.988235, + 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, 0.933333, 0.992157, + 0.941176, 0.247059, 0, 0, 0, 0, 0, 0, 0.188235, 0.898039, 0.992157, + 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.039216, 0.639216, 0.933333, + 0.988235, 0.913725, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0.113725, 0.843137, + 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.235294, 0.988235, + 0.992157, 0.988235, 0.815686, 0.07451, 0, 0, 0, 0, 0, 0, 0, 0.333333, + 0.988235, 0.988235, 0.552941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.211765, + 0.878431, 0.988235, 0.992157, 0.701961, 0.329412, 0.109804, 0, 0, 0, 0, 0, + 0, 0, 0.698039, 0.988235, 0.913725, 0.145098, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.188235, 0.890196, 0.988235, 0.988235, 0.745098, 0.047059, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0.882353, 0.988235, 0.568627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, + 0.933333, 0.992157, 0.992157, 0.992157, 0.447059, 0.294118, 0, 0, 0, 0, 0, + 0, 0, 0, 0.447059, 0.992157, 0.768627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.623529, 0.988235, 0.988235, 0.988235, 0.988235, 0.992157, 0.47451, 0, 0, + 0, 0, 0, 0, 0, 0.188235, 0.933333, 0.87451, 0.509804, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0.992157, 0.988235, 0.937255, 0.792157, 0.988235, 0.894118, + 0.082353, 0, 0, 0, 0, 0, 0, 0.027451, 0.647059, 0.992157, 0.654902, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.913725, 0.329412, 0.376471, + 0.184314, 0, 0, 0, 0, 0, 0, 0.027451, 0.513725, 0.988235, 0.635294, + 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.929412, 0.988235, + 0.988235, 0.741176, 0.309804, 0, 0, 0, 0, 0, 0, 0.529412, 0.988235, + 0.678431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.223529, 0.992157, + 0.992157, 1, 0.992157, 0.992157, 0.992157, 0.992157, 1, 0.992157, 0.992157, + 0.882353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.023529, + 0.478431, 0.654902, 0.658824, 0.952941, 0.988235, 0.988235, 0.988235, + 0.992157, 0.988235, 0.729412, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.647059, 0.764706, 0.764706, 0.768627, + 0.580392, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0 +]]] def main(): conf = parse_config("./mnist_model/trainer_config.py", "") print conf.data_config.load_data_args - network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) # For code hint. network.loadParameters("./mnist_model/") converter = DataProviderConverter([dense_vector(784)]) diff --git a/doc_cn/algorithm/rnn/hierarchical-layer.md b/doc_cn/algorithm/rnn/hierarchical-layer.md new file mode 100644 index 0000000000000..519653df081d6 --- /dev/null +++ b/doc_cn/algorithm/rnn/hierarchical-layer.md @@ -0,0 +1,66 @@ +# 支持双层序列作为输入的Layer + +## 概述 + +在自然语言处理任务中，序列是一种常见的数据类型。一个独立的词语，可以看作是一个非序列输入，或者，我们称之为一个0层的序列；由词语构成的句子，是一个单层序列；若干个句子构成一个段落，是一个双层的序列。 + +双层序列是一个嵌套的序列，它的每一个元素，又是一个单层的序列。这是一种非常灵活的数据组织方式，帮助我们构造一些复杂的输入信息。 + +我们可以按照如下层次定义非序列，单层序列，以及双层序列。 + ++ 0层序列：一个独立的元素，类型可以是PaddlePaddle支持的任意输入数据类型 ++ 单层序列：排成一列的多个元素，每个元素是一个0层序列，元素之间的顺序是重要的输入信息 ++ 双层序列：排成一列的多个元素，每个元素是一个单层序列，称之为双层序列的一个子序列（subseq），subseq的每个元素是一个0层序列 + + +在 PaddlePaddle中，下面这些Layer能够接受双层序列作为输入，完成相应的计算。 +## pooling_layer + +pooling_layer的使用示例如下，详细见配置API。 +```python +seq_pool = pooling_layer(input=layer, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) +``` +- `pooling_type` 目前支持两种，分别是：MaxPooling()和AvgPooling()。 +- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： + - 作用：双层序列经过运算变成一个0层序列，或单层序列经过运算变成一个0层序列 + - 输入：一个双层序列，或一个单层序列 + - 输出：一个0层序列，即整个输入序列（单层或双层）的平均值（或最大值） +- `agg_level=AggregateLevel.EACH_SEQUENCE`时： + - 作用：一个双层序列经过运算变成一个单层序列 + - 输入：必须是一个双层序列 + - 输出：一个单层序列，序列的每个元素是原来双层序列每个subseq元素的平均值（或最大值） + +## last_seq 和 first_seq + +last_seq的使用示例如下（first_seq类似），详细见配置API。 +```python +last = last_seq(input=layer, + agg_level=AggregateLevel.EACH_SEQUENCE) +``` +- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： + - 作用：一个双层序列经过运算变成一个0层序列，或一个单层序列经过运算变成一个0层序列 + - 输入：一个双层序列或一个单层序列 + - 输出：一个0层序列，即整个输入序列（双层或者单层）最后一个，或第一个元素。 +- `agg_level=AggregateLevel.EACH_SEQUENCE`时： + - 作用：一个双层序列经过运算变成一个单层序列 + - 输入：必须是一个双层序列 + - 输出：一个单层序列，其中每个元素是双层序列中每个subseq最后一个（或第一个）元素。 + +## expand_layer + +expand_layer的使用示例如下，详细见配置API。 +```python +expand = expand_layer(input=layer1, + expand_as=layer2, + expand_level=ExpandLevel.FROM_TIMESTEP) +``` +- `expand_level=ExpandLevel.FROM_TIMESTEP`时（默认值）： + - 作用：一个0层序列经过运算扩展成一个单层序列，或者一个双层序列 + - 输入：layer1必须是一个0层序列，是待扩展的数据；layer2可以是一个单层序列，或者是一个双层序列，提供扩展的长度信息 + - 输出：一个单层序列，或一个双层序列，输出序列的类型（双层序列，或单层序列）和序列中含有元素的数目同 layer2一致。若输出是单层序列，单层序列的每个元素（0层序列），都是对layer1元素的拷贝；若输出是双层序列，双层序列每个subseq中每个元素（0层序列），都是对layer1元素的拷贝 +- `expand_level=ExpandLevel.FROM_SEQUENCE`时： + - 作用：一个单层序列经过运算扩展成一个双层序列 + - 输入：layer1必须是一个单层序列，是待扩展的数据；layer2必须是一个双层序列，提供扩展的长度信息 + - 输出：一个双层序列，序列中含有元素的数目同layer2一致。要求单层序列含有元素的数目（0层序列），和双层序列含有subseq 的数目一致。单层序列第i个元素（0层序列），被扩展为一个单层序列，构成了输出双层序列的第i个subseq。 diff --git a/doc_cn/algorithm/rnn/hierarchical-rnn.md b/doc_cn/algorithm/rnn/hierarchical-rnn.md new file mode 100644 index 0000000000000..c184a34e85a57 --- /dev/null +++ b/doc_cn/algorithm/rnn/hierarchical-rnn.md @@ -0,0 +1,403 @@ +# 双层RNN配置与示例 + +我们在`paddle/gserver/tests/test_RecurrentGradientMachine`单测中，通过多组语义相同的单双层RNN配置，讲解如何使用双层RNN。 + +## 示例1：双进双出，subseq间无memory + +配置：单层RNN（`sequence_layer_group`）和双层RNN（`sequence_nest_layer_group`），语义完全相同。 + +### 读取双层序列的方法 + +首先，我们看一下单双层序列的不同数据组织形式（您也可以采用别的组织形式）： + +- 单层序列的数据（`Sequence/tour_train_wdseg`）如下，一共有10个样本。每个样本由两部分组成，一个label（此处都为2）和一个已经分词后的句子。 + +```text +2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 +2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * +2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 +2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 +2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . +2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 +2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ +2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 +2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * +2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 +``` + +- 双层序列的数据（`Sequence/tour_train_wdseg.nest`）如下，一共有4个样本。样本间用空行分开，代表不同的双层序列，序列数据和上面的完全一样。每个样本的子句数分别为2,3,2,3。 + +```text +2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 +2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * + +2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 +2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 +2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . + +2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 +2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ + +2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 +2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * +2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 +``` + +其次，我们看一下单双层序列的不同dataprovider（见`sequenceGen.py`）： + +- 单层序列的dataprovider如下： + - word_slot是integer_value_sequence类型，代表单层序列。 + - label是integer_value类型，代表一个向量。 + +```python +def hook(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sequence(len(settings.word_dict)), + integer_value(3)] + +@provider(init_hook=hook) +def process(settings, file_name): + with open(file_name, 'r') as fdata: + for line in fdata: + label, comment = line.strip().split('\t') + label = int(''.join(label.split())) + words = comment.split() + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + yield word_slot, label +``` + +- 双层序列的dataprovider如下： + - word_slot是integer_value_sub_sequence类型，代表双层序列。 + - label是integer_value_sequence类型，代表单层序列，即一个子句一个label。注意：也可以为integer_value类型，代表一个向量，即一个句子一个label。通常根据任务需求进行不同设置。 + - 关于dataprovider中input_types的详细用法，参见PyDataProvider2。 + +```python +def hook2(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), + integer_value_sequence(3)] + +@provider(init_hook=hook2) +def process2(settings, file_name): + with open(file_name) as fdata: + label_list = [] + word_slot_list = [] + for line in fdata: + if (len(line)) > 1: + label,comment = line.strip().split('\t') + label = int(''.join(label.split())) + words = comment.split() + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + label_list.append(label) + word_slot_list.append(word_slot) + else: + yield word_slot_list, label_list + label_list = [] + word_slot_list = [] +``` + +### 模型中的配置 + +首先，我们看一下单层序列的配置（见`sequence_layer_group.conf`）。注意：batchsize=5表示一次过5句单层序列，因此2个batch就可以完成1个pass。 + +```python +settings(batch_size=5) + +data = data_layer(name="word", size=dict_dim) + +emb = embedding_layer(input=data, size=word_dim) + +# (lstm_input + lstm) is equal to lstmemory +with mixed_layer(size=hidden_dim*4) as lstm_input: + lstm_input += full_matrix_projection(input=emb) + +lstm = lstmemory_group(input=lstm_input, + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + +lstm_last = last_seq(input=lstm) + +with mixed_layer(size=label_dim, + act=SoftmaxActivation(), + bias_attr=True) as output: + output += full_matrix_projection(input=lstm_last) + +outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) + +``` +其次，我们看一下语义相同的双层序列配置（见`sequence_nest_layer_group.conf`），并对其详细分析： + +- batchsize=2表示一次过2句双层序列。但从上面的数据格式可知，2句双层序列和5句单层序列的数据完全一样。 +- data_layer和embedding_layer不关心数据是否是序列格式，因此两个配置在这两层上的输出是一样的。 +- lstmemory: + - 单层序列过了一个mixed_layer和lstmemory_group。 + - 双层序列在同样的mixed_layer和lstmemory_group外，直接加了一层group。由于这个外层group里面没有memory，表示subseq间不存在联系，即起到的作用仅仅是把双层seq拆成单层，因此双层序列过完lstmemory的输出和单层的一样。 +- last_seq： + - 单层序列直接取了最后一个元素 + - 双层序列首先（last_seq层）取了每个subseq的最后一个元素，将其拼接成一个新的单层序列；接着（expand_layer层）将其扩展成一个新的双层序列，其中第i个subseq中的所有向量均为输入的单层序列中的第i个向量；最后（average_layer层）取了每个subseq的平均值。 + - 分析得出：第一个last_seq后，每个subseq的最后一个元素就等于单层序列的最后一个元素，而expand_layer和average_layer后，依然保持每个subseq最后一个元素的值不变（这两层仅是为了展示它们的用法，实际中并不需要）。因此单双层序列的输出是一样旳。 + +```python +settings(batch_size=2) + +data = data_layer(name="word", size=dict_dim) + +emb_group = embedding_layer(input=data, size=word_dim) + +# (lstm_input + lstm) is equal to lstmemory +def lstm_group(lstm_group_input): + with mixed_layer(size=hidden_dim*4) as group_input: + group_input += full_matrix_projection(input=lstm_group_input) + + lstm_output = lstmemory_group(input=group_input, + name="lstm_group", + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + return lstm_output + +lstm_nest_group = recurrent_group(input=SubsequenceInput(emb_group), + step=lstm_group, + name="lstm_nest_group") +# hasSubseq ->(seqlastins) seq +lstm_last = last_seq(input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) + +# seq ->(expand) hasSubseq +lstm_expand = expand_layer(input=lstm_last, expand_as=emb_group, expand_level=ExpandLevel.FROM_SEQUENCE) + +# hasSubseq ->(average) seq +lstm_average = pooling_layer(input=lstm_expand, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) + +with mixed_layer(size=label_dim, + act=SoftmaxActivation(), + bias_attr=True) as output: + output += full_matrix_projection(input=lstm_average) + +outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +``` +## 示例2：双进双出，subseq间有memory + +配置：单层RNN（`sequence_rnn.conf`），双层RNN（`sequence_nest_rnn.conf`和`sequence_nest_rnn_readonly_memory.conf`），语义完全相同。 + +### 读取双层序列的方法 + +我们看一下单双层序列的不同数据组织形式和dataprovider（见`rnn_data_provider.py`） +```python +data = [ + [[[1, 3, 2], [4, 5, 2]], 0], + [[[0, 2], [2, 5], [0, 1, 2]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value(3)]) +def process_subseq(settings, file_name): + for d in data: + yield d + +@provider(input_types=[integer_value_sequence(10), + integer_value(3)]) +def process_seq(settings, file_name): + for d in data: + seq = [] +``` +- 单层序列：有两句，分别为[1,3,2,4,5,2]和[0,2,2,5,0,1,2]。 +- 双层序列：有两句，分别为[[1,3,2],[4,5,2]]（2个子句）和[[0,2],[2,5],[0,1,2]]（3个子句）。 +- 单双层序列的label都分别是0和1 + +### 模型中的配置 + +我们选取单双层序列配置中的不同部分，来对比分析两者语义相同的原因。 + +- 单层序列：过了一个很简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全链接。 + +```python +def step(y): + mem = memory(name="rnn_state", size=hidden_dim) + return fc_layer(input=[y, mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="rnn_state") + +out = recurrent_group(step=step, input=emb) +``` +- 双层序列，外层memory是一个元素： + - 内层inner_step的recurrent_group和单层序列的几乎一样。除了boot_layer=outer_mem，表示将外层的outer_mem作为内层memory的初始状态。外层outer_step中，outer_mem是一个子句的最后一个向量，即整个双层group是将前一个子句的最后一个向量，作为下一个子句memory的初始状态。 + - 从输入数据上看，单双层序列的句子是一样的，只是双层序列将其又做了子序列划分。因此双层序列的配置中，必须将前一个子句的最后一个元素，作为boot_layer传给下一个子句的memory，才能保证和单层序列的配置中“每一个时间步都用了上一个时间步的输出结果”一致。 + +```python +def outer_step(x): + outer_mem = memory(name="outer_rnn_state", size=hidden_dim) + def inner_step(y): + inner_mem = memory(name="inner_rnn_state", + size=hidden_dim, + boot_layer=outer_mem) + return fc_layer(input=[y, inner_mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="inner_rnn_state") + + inner_rnn_output = recurrent_group( + step=inner_step, + input=x) + last = last_seq(input=inner_rnn_output, name="outer_rnn_state") + + return inner_rnn_output + +out = recurrent_group(step=outer_step, input=SubsequenceInput(emb)) +``` +- 双层序列，外层memory是单层序列： + - 由于外层每个时间步返回的是一个子句，这些子句的长度往往不等长。因此当外层有is_seq=True的memory时，内层是**无法直接使用**它的，即内层memory的boot_layer不能链接外层的这个memory。 + - 如果内层memory想**间接使用**这个外层memory，只能通过`pooling_layer`、`last_seq`或`first_seq`这三个layer将它先变成一个元素。但这种情况下，外层memory必须有boot_layer，否则在第0个时间步时，由于外层memory没有任何seq信息，因此上述三个layer的前向会报出“**Check failed: input.sequenceStartPositions**”的错误。 + +## 示例3：双进双出，输入不等长 + +**输入不等长**是指recurrent_group的多个输入在各时刻的长度可以不相等, 但需要指定一个和输出长度一致的input，用targetInlink表示。参考配置：单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`），双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`） + +### 读取双层序列的方法 + +我们看一下单双层序列的数据组织形式和dataprovider（见`rnn_data_provider.py`） +```python +data2 = [ + [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], + [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value_sub_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_subseq(settings, file_name): #双层RNN的dataprovider + for d in data2: + yield d + + +@provider(input_types=[integer_value_sequence(10), + integer_value_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_seq(settings, file_name): #单层RNN的dataprovider + for d in data2: + words1=reduce(lambda x,y: x+y, d[0]) + words2=reduce(lambda x,y: x+y, d[1]) + yield words1, words2, d[2] +``` + +data2 中有两个样本，每个样本有两个特征, 记fea1, fea2。 + +- 单层序列：两个样本分别为[[1, 2, 4, 5, 2], [5, 4, 1, 3, 1]] 和 [[0, 2, 2, 5, 0, 1, 2], [1, 5, 4, 2, 3, 6, 1]] +- 双层序列：两个样本分别为 + - **样本1**：[[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]]]。fea1和fea2都分别有2个子句，fea1=[[1, 2], [4, 5, 2]], fea2=[[5, 4, 1], [3, 1]] + - **样本2**：[[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]]]。fea1和fea2都分别有3个子句， fea1=[[0, 2], [2, 5], [0, 1, 2]], fea2=[[1, 5], [4], [2, 3, 6, 1]]。
+ - **注意**：每个样本中，各特征的子句数目需要相等。这里说的“双进双出，输入不等长”是指fea1在i时刻的输入的长度可以不等于fea2在i时刻的输入的长度。如对于第1个样本，时刻i=2, fea1[2]=[4, 5, 2]，fea2[2]=[3, 1]，3≠2。 +- 单双层序列中，两个样本的label都分别是0和1 + +### 模型中的配置 + +单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`）和双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`）两个模型配置达到的效果完全一样，区别只在于输入为单层还是双层序列，现在我们来看它们内部分别是如何实现的。 + +- 单层序列： + - 过了一个简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全连接，功能与示例2中`sequence_rnn.conf`的`step`函数完全相同。这里，两个输入x1,x2分别通过calrnn返回最后时刻的状态。结果得到的encoder1_rep和encoder2_rep分别是单层序列，最后取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 + - 注意到这里recurrent_group输入的每个样本中，fea1和fea2的长度都分别相等，这并非偶然，而是因为recurrent_group要求输入为单层序列时，所有输入的长度都必须相等。 + +```python +def step(x1, x2): + def calrnn(y): + mem = memory(name = 'rnn_state_' + y.name, size = hidden_dim) + out = fc_layer(input = [y, mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'rnn_state_' + y.name) + return out + + encoder1 = calrnn(x1) + encoder2 = calrnn(x2) + return [encoder1, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="stepout", + step=step, + input=[emb1, emb2]) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) +``` +- 双层序列： + - 双层RNN中，对输入的两个特征分别求时序上的连续全连接(`inner_step1`和`inner_step2`分别处理fea1和fea2)，其功能与示例2中`sequence_nest_rnn.conf`的`outer_step`函数完全相同。不同之处是，此时输入`[SubsequenceInput(emb1), SubsequenceInput(emb2)]`在各时刻并不等长。 + - 函数`outer_step`中可以分别处理这两个特征，但我们需要用targetInlink指定recurrent_group的输出的格式（各子句长度）只能和其中一个保持一致，如这里选择了和emb2的长度一致。 + - 最后，依然是取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 + +```python +def outer_step(x1, x2): + outer_mem1 = memory(name = "outer_rnn_state1", size = hidden_dim) + outer_mem2 = memory(name = "outer_rnn_state2", size = hidden_dim) + def inner_step1(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem1) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + def inner_step2(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem2) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + encoder1 = recurrent_group( + step = inner_step1, + name = 'inner1', + input = x1) + + encoder2 = recurrent_group( + step = inner_step2, + name = 'inner2', + input = x2) + + sentence_last_state1 = last_seq(input = encoder1, name = 'outer_rnn_state1') + sentence_last_state2_ = last_seq(input = encoder2, name = 'outer_rnn_state2') + + encoder1_expand = expand_layer(input = sentence_last_state1, + expand_as = encoder2) + + return [encoder1_expand, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="outer", + step=outer_step, + input=[SubsequenceInput(emb1), SubsequenceInput(emb2)], + targetInlink=emb2) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) +``` + +## 示例4：beam_search的生成 + +TBD diff --git a/doc_cn/algorithm/rnn/rnn-tutorial.md b/doc_cn/algorithm/rnn/rnn-tutorial.md new file mode 100644 index 0000000000000..9e488b0d51956 --- /dev/null +++ b/doc_cn/algorithm/rnn/rnn-tutorial.md @@ -0,0 +1,96 @@ +# Recurrent Group教程 + +## 概述 + +序列数据是自然语言处理任务面对的一种主要输入数据类型。 + +一句话是由词语构成的序列，多句话进一步构成了段落。因此，段落可以看作是一个嵌套的双层的序列，这个序列的每个元素又是一个序列。 + +双层序列是PaddlePaddle支持的一种非常灵活的数据组织方式，帮助我们更好地描述段落、多轮对话等更为复杂的语言数据。基于双层序列输入，我们可以设计搭建一个灵活的、层次化的RNN，分别从词语和句子级别编码输入数据，同时也能够引入更加复杂的记忆机制，更好地完成一些复杂的语言理解任务。 + +在PaddlePaddle中，`recurrent_group`是一种任意复杂的RNN单元，用户只需定义RNN在一个时间步内完成的计算，PaddlePaddle负责完成信息和误差在时间序列上的传播。 + +更进一步，`recurrent_group`同样可以扩展到双层序列的处理上。通过两个嵌套的`recurrent_group`分别定义子句级别和词语级别上需要完成的运算，最终实现一个层次化的复杂RNN。 + +目前，在PaddlePaddle中，能够对双向序列进行处理的有`recurrent_group`和部分Layer，具体可参考文档：支持双层序列作为输入的Layer。 + +## 相关概念 + +### 基本原理 +`recurrent_group` 是PaddlePaddle支持的一种任意复杂的RNN单元。使用者只需要关注于设计RNN在一个时间步之内完成的计算，PaddlePaddle负责完成信息和梯度在时间序列上的传播。 + +PaddlePaddle中，`recurrent_group`的一个简单调用如下： + +``` python +recurrent_group(step, input, reverse) +``` +- step：一个可调用的函数，定义一个时间步之内RNN单元完成的计算 +- input：输入，必须是一个单层序列，或者一个双层序列 +- reverse：是否以逆序处理输入序列 + +使用`recurrent_group`的核心是设计step函数的计算逻辑。step函数内部可以自由组合PaddlePaddle支持的各种layer，完成任意的运算逻辑。`recurrent_group` 的输入（即input）会成为step函数的输入，由于step 函数只关注于RNN一个时间步之内的计算，在这里`recurrent_group`替我们完成了原始输入数据的拆分。 + +### 输入 +`recurrent_group`处理的输入序列主要分为以下三种类型： + +- **数据输入**：一个双层序列进入`recurrent_group`会被拆解为一个单层序列，一个单层序列进入`recurrent_group`会被拆解为非序列，然后交给step函数，这一过程对用户是完全透明的。可以有以下两种：1）通过data_layer拿到的用户输入；2）其它layer的输出。 + +- **只读Memory输入**：`StaticInput` 定义了一个只读的Memory，由`StaticInput`指定的输入不会被`recurrent_group`拆解，`recurrent_group` 循环展开的每个时间步总是能够引用所有输入，可以是一个非序列，或者一个单层序列。 + +- **序列生成任务的输入**：`GeneratedInput`只用于在序列生成任务中指定输入数据。 + +### 输入示例 + +序列生成任务大多遵循encoder-decoer架构，encoder和decoder可以是能够处理序列的任意神经网络单元，而RNN是最流行的选择。 + +给定encoder输出和当前词，decoder每次预测产生下一个最可能的词语。在这种结构中，decoder接受两个输入： + +- 要生成的目标序列：是decoder的数据输入，也是decoder循环展开的依据，`recurrent_group`会对这类输入进行拆解。 + +- encoder输出，可以是一个非序列，或者一个单层序列：是一个unbounded memory，decoder循环展开的每一个时间步会引用全部结果，不应该被拆解，这种类型的输入必须通过`StaticInput`指定。关于Unbounded Memory的更多讨论请参考论文 [Neural Turning Machine](https://arxiv.org/abs/1410.5401)。 + +在序列生成任务中，decoder RNN总是引用上一时刻预测出的词的词向量，作为当前时刻输入。`GeneratedInput`自动完成这一过程。 + +### 输出 +`step`函数必须返回一个或多个Layer的输出，这个Layer的输出会作为整个`recurrent_group` 最终的输出结果。在输出的过程中，`recurrent_group` 会将每个时间步的输出拼接，这个过程对用户也是透明的。 + +### memory +memory只能在`recurrent_group`中定义和使用。memory不能独立存在，必须指向一个PaddlePaddle定义的Layer。引用memory得到这layer上一时刻输出，因此，可以将memory理解为一个时延操作。 + +可以显示地指定一个layer的输出用于初始化memory。不指定时，memory默认初始化为0。 + +## 双层RNN介绍 +`recurrent_group`帮助我们完成对输入序列的拆分，对输出的合并，以及计算逻辑在序列上的循环展开。 + +利用这种特性，两个嵌套的`recurrent_group`能够处理双层序列，实现词语和句子两个级别的双层RNN结构。 + +- 单层（word-level）RNN：每个状态（state）对应一个词（word）。 +- 双层（sequence-level）RNN：一个双层RNN由多个单层RNN组成，每个单层RNN（即双层RNN的每个状态）对应一个子句（subseq）。 + +为了描述方便，下文以NLP任务为例，将含有子句（subseq）的段落定义为一个双层序列，将含有词语的句子定义为一个单层序列，那么0层序列即为一个词语。 + +## 双层RNN的使用 + +### 训练流程的使用方法 +使用 `recurrent_group`需要遵循以下约定： + +- **单进单出**：输入和输出都是单层序列。 + - 如果有多个输入，不同输入序列含有的词语数必须严格相等。 + - 输出一个单层序列，输出序列的词语数和输入序列一致。 + - memory：在step函数中定义 memory指向一个layer，通过引用memory得到这个layer上一个时刻输出，形成recurrent 连接。memory的is_seq参数必须为false。如果没有定义memory，每个时间步之内的运算是独立的。 + - boot_layer：memory的初始状态，默认初始状为0，memory的is_seq参数必须为false。 + +- **双进双出**：输入和输出都是双层序列。 + - 如果有多个输入序列，不同输入含有的子句（subseq）数必须严格相等，但子句含有的词语数可以不相等。 + - 输出一个双层序列，子句（subseq）数、子句的单词数和指定的一个输入序列一致，默认为第一个输入。 + - memory：在step函数中定义memory，指向一个layer，通过引用memory得到这个layer上一个时刻的输出，形成recurrent连接。定义在外层`recurrent_group` step函数中的memory，能够记录上一个subseq 的状态，可以是一个单层序列（只作为read-only memory），也可以是一个词语。如果没有定义memory，那么 subseq 之间的运算是独立的。 + - boot_layer：memory 初始状态，可以是一个单层序列（只作为read-only memory）或一个向量。默认不设置，即初始状态为0。 + +- **双进单出**：目前还未支持，会报错"In hierachical RNN, all out links should be from sequences now"。 + + +### 生成流程的使用方法 +使用`beam_search`需要遵循以下约定： + +- 单层RNN：从一个word生成下一个word。 +- 双层RNN：即把单层RNN生成后的subseq给拼接成一个新的双层seq。从语义上看，也不存在一个subseq直接生成下一个subseq的情况。 diff --git a/doc_cn/build_and_install/index.rst b/doc_cn/build_and_install/index.rst index e21fc98c63dcd..2205e282248c4 100644 --- a/doc_cn/build_and_install/index.rst +++ b/doc_cn/build_and_install/index.rst @@ -1,19 +1,32 @@ 编译与安装 ======================== -PaddlePaddle提供数个预编译的二进制来进行安装，包括Docker镜像，ubuntu的deb安装包等。我们推荐使用Docker镜像来部署环境，同时欢迎贡献更多的安装包。 - -Note: The intallation packages are still in pre-release state and your experience of installation may not be smooth. +安装 +++++ -注意：目前PaddlePaddle的安装包还处在pre-release的状态，使用起来或许会不是很顺畅。 +PaddlePaddle提供数个预编译的二进制来进行安装，包括Docker镜像，ubuntu的deb安装包等。我们推荐使用Docker镜像来部署环境，同时欢迎贡献更多的安装包。 .. toctree:: :maxdepth: 1 :glob: - 源码下载(对内) <../build/internal/download_paddle_source_zh_cn.rst> 使用Jumbo安装(对内) <../build/internal/install_from_jumbo.rst> - 从源码编译安装(对内) <../build/internal/build_from_source_zh_cn.rst> install/docker_install.rst install/ubuntu_install.rst + + + +编译 +++++ + +.. warning:: + + 编译选项主要推荐高级用户查看，普通用户请走安装流程。 + +.. toctree:: + :maxdepth: 1 + :glob: + + 源码下载(对内) <../build/internal/download_paddle_source_zh_cn.rst> + 从源码编译安装(对内) <../build/internal/build_from_source_zh_cn.rst> cmake/index.rst diff --git a/doc_cn/build_and_install/install/docker_install.rst b/doc_cn/build_and_install/install/docker_install.rst index 4154cb86d8d64..a5f5fb117e11e 100644 --- a/doc_cn/build_and_install/install/docker_install.rst +++ b/doc_cn/build_and_install/install/docker_install.rst @@ -14,20 +14,43 @@ PaddlePaddle提供了Docker的使用镜像。PaddlePaddle推荐使用Docker进 PaddlePaddle提供的Docker镜像版本 -------------------------------- -我们提供了6个Docker image\: +我们提供了12个 `Docker image `_ ，他们的image name都是 :code:`paddle-dev/paddle` ，tag分别为 -* paddledev/paddle\:cpu-latest\: PaddlePaddle的CPU二进制 -* paddledev/paddle\:gpu-latest\： PaddlePaddle的GPU二进制 -* paddledev/paddle\:cpu-devel-latest\: PaddlePaddle的CPU二进制,同时包含CPU开发环境和源码 -* paddledev/paddle\:gpu-devel-latest\: PaddlePaddle的GPU二进制,同时包含GPU开发环境和源码 -* paddledev/paddle\:cpu-demo-latest\: PaddlePaddle的CPU二进制,同时包含CPU开发环境、源码和运行demo的必要依赖 -* paddledev/paddle\:gpu-demo-latest\: PaddlePaddle的GPU二进制,同时包含GPU开发环境、源码和运行demo的必要依赖 ++-----------------+------------------+------------------------+-----------------------+ +| | normal | devel | demo | ++=================+==================+========================+=======================+ +| CPU | cpu-latest | cpu-devel-latest | cpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU | gpu-latest | gpu-devel-latest | gpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| CPU WITHOUT AVX | cpu-noavx-latest | cpu-noavx-devel-latest | cpu-noavx-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU WITHOUT AVX | gpu-noavx-latest | gpu-noavx-devel-latest | gpu-noavx-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ -同时，不同的稳定版本，会将latest替换成稳定版本的版本号。 +其中，横向包括三个版本，normal，devel和demo。 + +* Normal: 正常的Docker image，只包括paddle的二进制 +* Devel: 包括Paddle的二进制、编译环境和源代码 +* Demo: 包括Paddle运行demo所需要的依赖 + +纵向包括四个版本，他们是。 + +* CPU: CPU版本。需要支持AVX指令集的CPU +* GPU: GPU版本。需要支持AVX指令集的CPU +* CPU WITHOUT AVX: CPU版本，不支持AVX指令集的CPU也可以运行 +* GPU WITHOUT AVX: GPU版本，不需要AVX指令集的CPU也可以运行。 + +用户可以选择对应版本的docker image。使用如下脚本可以确定本机的CPU知否支持 :code:`AVX` 指令集\: + +.. code-block:: bash + + if cat /proc/cpuinfo | grep -q avx ; then echo "Support AVX"; else echo "Not support AVX"; fi + +如果输出 :code:`Support AVX`，则可以选择上表中的AVX版本PaddlePaddle。否则需要选择非AVX的PaddlePaddle。选择普通CPU版本的devel版本的image，则可以使用 :code:`paddle-dev/paddle:cpu-devel-latest` 来引用这个image。 PaddlePaddle提供的镜像并不包含任何命令运行，想要运行PaddlePaddle，您需要进入镜像运行PaddlePaddle -程序或者自定义一个含有启动脚本的image。具体请参考注意事项中的 -`使用ssh访问PaddlePaddle镜像` +程序或者自定义一个含有启动脚本的image。具体请参考注意事项中的 :code:`使用ssh访问PaddlePaddle镜像` 下载和运行Docker镜像 -------------------- @@ -44,7 +67,7 @@ mac osx或者是windows机器，请参考 .. code-block:: bash - $ docker run -it paddledev/paddlepaddle:latest-cpu + $ docker run -it paddledev/paddlepaddle:cpu-latest 即可启动和进入PaddlePaddle的container。如果运行GPU版本的PaddlePaddle，则需要先将 cuda相关的Driver和设备映射进container中，脚本类似于 @@ -53,7 +76,7 @@ cuda相关的Driver和设备映射进container中，脚本类似于 $ export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - $ docker run -it paddledev/paddlepaddle:latest-gpu + $ docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddlepaddle:latest-gpu 进入Docker container后，运行 :code:`paddle version` 即可打印出PaddlePaddle的版本和构建 信息。安装完成的PaddlePaddle主体包括三个部分， :code:`paddle` 脚本， python的 diff --git a/doc_cn/build_and_install/install/paddle_version.txt b/doc_cn/build_and_install/install/paddle_version.txt new file mode 100644 index 0000000000000..a80873303fd0d --- /dev/null +++ b/doc_cn/build_and_install/install/paddle_version.txt @@ -0,0 +1,11 @@ +PaddlePaddle 0.8.0b1, compiled with + with_avx: ON + with_gpu: OFF + with_double: OFF + with_python: ON + with_rdma: OFF + with_glog: ON + with_gflags: ON + with_metric_learning: + with_timer: OFF + with_predict_sdk: diff --git a/doc_cn/build_and_install/install/ubuntu_install.rst b/doc_cn/build_and_install/install/ubuntu_install.rst index 7cdd470677eb8..0fb59e25f6932 100644 --- a/doc_cn/build_and_install/install/ubuntu_install.rst +++ b/doc_cn/build_and_install/install/ubuntu_install.rst @@ -1,29 +1,38 @@ 使用deb包在Ubuntu上安装PaddlePaddle =================================== -PaddlePaddle目前支持ubuntu 14.04版本使用deb包安装。更多的安装包PaddlePaddle会在近期提供。 -欢迎大家贡献各个发行版的安装包(例如，ubuntu，centos，debian，gentoo)。 +PaddlePaddle目前支持使用deb包安装。Paddle的 :code:`deb` 安装包在ubuntu 14.04中正确，但理论上支持其他的 debian 发行版。 -PaddlePaddle的ubuntu安装包分为两个版本，即CPU版本，和GPU版本，他们的下载地址是\: -https://github.com/baidu/Paddle/releases/tag/V0.8.0b0 -需要注意的是，目前PaddlePaddle的安装包只支持 -`AVX `_ -指令集的X86 CPU。如果系统使用不支持 `AVX`_ 指令集的CPU运行PaddlePaddle，那么需要从源码 -编译PaddlePaddle，请参考 `编译文档 <../cmake/index.html>`_ 。 +PaddlePaddle的ubuntu安装包分为四个版本，他们是 cpu、gpu、cpu-noavx、gpu-noavx 四个版本。其中 noavx 用于不支持AVX指令集的cpu。安装包的下载地址是\: https://github.com/baidu/Paddle/releases/ -用户需要先将PaddlePaddle安装包下载到本地，然后执行如下命令即可完成安装。 + +用户需要先将PaddlePaddle安装包下载到本地，然后执行如下 :code:`gdebi` 命令即可完成安装。 + +.. code-block:: shell + + gdebi paddle-*-cpu*.deb + +如果 :code:`gdebi` 没有安装,则需要使用 :code:`sudo apt-get install gdebi`, 来安装 :code:`gdebi` 。 + + +或者使用下面一条命令安装. .. code-block:: shell - dpkg -i paddle-0.8.0b-cpu.deb + dpkg -i paddle-*-cpu*.deb apt-get install -f 在 :code:`dpkg -i` 的时候如果报一些依赖未找到的错误是正常的， 在 :code:`apt-get install -f` 里会继续安装 PaddlePaddle。 + 需要注意的是，如果使用GPU版本的PaddlePaddle，请安装CUDA 7.5 和CUDNN 5到本地环境中， 并设置好对应的环境变量(LD_LIBRARY_PATH等等)。 +安装完成后,可以使用命令 :code:`paddle version` 查看安装后的paddle 版本。可能的输出为 + +.. literalinclude:: paddle_version.txt + 可能遇到的问题 -------------- diff --git a/doc_cn/concepts/nn.rst b/doc_cn/concepts/nn.rst new file mode 100644 index 0000000000000..f4d2cf490d147 --- /dev/null +++ b/doc_cn/concepts/nn.rst @@ -0,0 +1,3 @@ +TBD + +目前正在书写中。敬请期待。 \ No newline at end of file diff --git a/doc_cn/concepts/program_concepts.rst b/doc_cn/concepts/program_concepts.rst new file mode 100644 index 0000000000000..af5bbdac260af --- /dev/null +++ b/doc_cn/concepts/program_concepts.rst @@ -0,0 +1,4 @@ +TBD +### + +目前正在书写中。敬请期待。 \ No newline at end of file diff --git a/doc_cn/concepts/pserver_topology.dot b/doc_cn/concepts/pserver_topology.dot new file mode 100644 index 0000000000000..9ff658b849503 --- /dev/null +++ b/doc_cn/concepts/pserver_topology.dot @@ -0,0 +1,68 @@ +graph pp_topology { + rankdir=BT; + subgraph cluster_node0 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器0" + + pserver0 [label="Parameter \n Server 0"] + trainer0 [label="Trainer 0"] + } + subgraph cluster_node1 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器1" + + pserver1 [label="Parameter \n Server 1"] + trainer1 [label="Trainer 1"] + } + + subgraph cluster_node2 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器2" + + pserver2 [label="Parameter \n Server 2"] + trainer2 [label="Trainer 2"] + } + + subgraph cluster_node3 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器3" + + pserver3 [label="Parameter \n Server 3"] + trainer3 [label="Trainer 3"] + } + + data [label="数据", shape=hexagon] + + trainer0 -- pserver0 + trainer0 -- pserver1 + trainer0 -- pserver2 + trainer0 -- pserver3 + + trainer1 -- pserver0 + trainer1 -- pserver1 + trainer1 -- pserver2 + trainer1 -- pserver3 + + trainer2 -- pserver0 + trainer2 -- pserver1 + trainer2 -- pserver2 + trainer2 -- pserver3 + + trainer3 -- pserver0 + trainer3 -- pserver1 + trainer3 -- pserver2 + trainer3 -- pserver3 + + data -- trainer0 + data -- trainer1 + data -- trainer2 + data -- trainer3 +} diff --git a/doc_cn/concepts/trainer_config.py b/doc_cn/concepts/trainer_config.py new file mode 100644 index 0000000000000..3eccbd7bc11f4 --- /dev/null +++ b/doc_cn/concepts/trainer_config.py @@ -0,0 +1,29 @@ +from paddle.trainer_config_helpers import * + +define_py_data_sources2( + train_list='train.list', + test_list='test.list', + module='provider', + obj='process') +settings( + batch_size=128, + learning_rate=1e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(0.5)) + +img = data_layer(name='pixel', size=28 * 28) + +hidden1 = simple_img_conv_pool( + input=img, filter_size=3, num_filters=32, pool_size=3, num_channel=1) + +hidden2 = fc_layer( + input=hidden1, + size=200, + act=TanhActivation(), + layer_attr=ExtraAttr(drop_rate=0.5)) +predict = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) + +outputs( + classification_cost( + input=predict, label=data_layer( + name='label', size=10))) diff --git a/doc_cn/concepts/use_concepts.rst b/doc_cn/concepts/use_concepts.rst new file mode 100644 index 0000000000000..67e98edabc0c2 --- /dev/null +++ b/doc_cn/concepts/use_concepts.rst @@ -0,0 +1,191 @@ +######################### +PaddlePaddle 基本使用概念 +######################### + +PaddlePaddle是一个神经网络学习框架。其单机进程为 :code:`paddle train`。 单机的所有设备使用，均在单机进程内调度完成。 而多机辅助进程 :code:`paddle pserver` 负责联合多个单机进程进行通信，进而充分利用集群的计算资源。 PaddlePaddle同时以 :code:`swig api` 的形式，提供训练结果模型预测的方法和自定义训练流程。 + +下面我们会分别介绍主要进程 :code:`paddle train` 中的一些概念。这些概念会对如何使用PaddlePaddle有一定的帮助。 了解这些概念的前提是，读者已经了解 `基本的神经网络/机器学习原理和概念 `_ 。同时，如果想要了解PaddlePaddle实现中的一些概念，请参考 `PaddlePaddle 编程中的基本概念 `_ 。 + +.. contents:: + +PaddlePaddle 的进程模型 +======================= + +PaddlePaddle进程内嵌了一个 :code:`python` 解释器。 这个 :code:`python` 解释器负责解析用户定义的神经网络配置，和解析用户数据，并将用户数据传入给 PaddlePaddle。 + +.. graphviz:: + + digraph pp_process { + rankdir=LR; + config_file [label="用户神经网络配置"]; + subgraph cluster_pp { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "PaddlePaddle C++"; + py [label="Python解释器"]; + } + data_provider [label="用户数据解析"]; + config_file -> py; + py -> data_provider [dir="back"]; + } + +所以，PaddlePaddle单机训练进程，:code:`paddle train` , 对于用户的主要接口语言为 python。 主要需要用户配置的两个文件为 :code:`DataProvider` 和训练文件 :code:`TrainerConfig` 。 + + +DataProvider +============ + +DataProvider是 :code:`paddle train` 的数据提供器。 它负责将用户的原始数据转换成 PaddlePaddle 可以识别的数据类型。每当 PaddlePaddle 需要新的数据训练时，都会调用 DataProvider 返回数据。 当所有数据读取完一轮后，DataProvider 便返回空数据通知 PaddlePaddle。PaddlePaddle负责在下一轮训练开始前，将DataProvider重置。 + +需要注意的是，DataProvider在PaddlePaddle中是被训练逻辑调用的关系， 而不是新的数据驱动训练。并且所有的 :code:`shuffle` , 和一些随机化的噪声添加，都应该在 DataProvider 阶段完成。 + +为了方便用户使用自己的数据格式， PaddlePaddle 提供了 `PyDataProvider`_ 来处理数据。 并且在这个Provider中，PaddlePaddle的 C++ 部分接管了如何shuffle，处理 batch，GPU/CPU通信，双缓冲，异步读取等问题。 用户可以参考 `PyDataProvider`_ 的相关文档，继续深入了解 DataProvider 的使用。 + + +训练文件 +======== + +训练文件是PaddlePaddle中配置神经网络结构、学习优化算法、数据传入方式的地方。 训练文件是一个python文件，使用命令行参数 :code:`--config` 传给 paddle 的主程序。 例如\: + +.. code-block:: bash + + paddle train --config=trainer_config.py + +一个典型简单的训练文件可能为 + +.. literalinclude:: trainer_config.py + :linenos: + +下面我们详细的介绍一下训练文件中各个模块的概念。 + + +trainer_config_helpers +---------------------- + +PaddlePaddle的配置文件与PaddlePaddle C++端通信的最基础协议是 :code:`protobuf` 。而为了避免用户直接写比较难写的 protobuf string，我们书写了一个helpers来生成这个protobuf包。所以在文件的开始，import这些helpers函数。 + +需要注意的是，这个 :code:`paddle.trainer_config_helpers` 包是标准的python包，这意味着用户可以选择自己喜欢的 :code:`ide` 或者编辑器来编写Paddle的配置文件，这个python包注释文档比较完善，并且考虑了IDE的代码提示与类型注释。 + +data_sources +------------ + +data_sources是配置神经网络的数据源。这里使用的函数是 :code:`define_py_data_sources2` ，这个函数是定义了使用 `PyDataProvider`_ 作为数据源。 而后缀 :code:`2` 是Paddle历史遗留问题，因为Paddle之前使用的 PyDataProvider 性能较差，所以完全重构了一个新的 `PyDataProvider`_ 。 + +data_sources里面的 train_list 和 test_list 指定的是训练文件列表和测试文件列表。 如果传入一个字符串的话，是指一个训练列表文件。这个训练列表文件中包含的是每一个训练或者测试文件的路径。如果传入一个list的话，则会默认生成一个 list 文件，再传入给 train.list 或者 test.list 。 + +而 :code:`module` 和 :code:`obj` 指定了 DataProvider 的模块名和函数名。 + +更具体的使用，请参考 `PyDataProvider`_ 。 + +settings +-------- + +`settings`_ 是神经网络训练算法相关的设置项。包括学习率，batch_size，优化算法，正则方法等等。具体的使用方法请参考 `settings`_ 文档。 + +网络配置 +-------- + +上述网络配置中余下的部分均是神经网络配置。第一行是定义一个名字叫 "pixel" 的 :code:`data_layer` 。每一个layer返回的都是一个 :code:`LayerOutput` 对象。 这里第一层的输出对象是 :code:`img` 。然后这个对象传输给了另一个 layer 函数， +:code:`simple_img_conv_pool` 。:code:`simple_img_conv_pool` 是一个组合层， +包括了图像的卷积 (convolution) 和池化(pooling)， +并继续接了一个全连接层( :code:`fc_layer` )，然后再接了一个Softmax的全连接层。 + +最终，网络配置输出了 :code:`classification_cost` 。标记网络输出的函数为 +:code:`outputs` 。网络的输出是神经网络的优化目标，神经网络训练的时候，实际上就是 +要最小化这个输出。 + +在神经网络进行预测的时候，实际上网络的输出也是通过 :code:`outputs` 标记。 + + +Layer、Projection、Operator +=========================== + +PaddlePaddle的网络基本上是基于Layer来配置的。所谓的Layer即是神经网络的某一层， +而神经网络的某一层，一般是封装了许多复杂操作的操作集合。比如最简单的 +:code:`fc_layer` ，也包括矩阵乘法，多输入的求和，和activation。 + +.. code-block:: python + + data = data_layer(name='data', size=200) + out = fc_layer(input=data, size=200, act=TanhActivation()) + +而对于更灵活配置需求，可能这样基于Layer的配置是不灵活的。于是 PaddlePaddle 提供 +了基于 Projection 或者 Operator 的配置。使用Projection和Operator需要与 +:code:`mixed_layer` 配合使用。 :code:`mixed_layer` 是将layer中的元素累加求和， +并且做一个 :code:`activation` ， 而这个layer具体如何计算，是交由内部的Projection +和 Operator 定义。Projection是指含有可学习参数的操作，而Operator不含有可学习的 +参数，输入全是其他Layer的输出。 + + +例如，和 :code:`fc_layer` 同样功能的 :code:`mixed_layer` 。 + +.. code-block:: python + + data = data_layer(name='data', size=200) + with mixed_layer(size=200) as out: + out += full_matrix_projection(input=data) + +PaddlePaddle可以使用的mixed layer 配置出非常复杂的网络，甚至可以直接配置一个完整的LSTM。 +用户可以参考 `mixed_layer`_ 的相关文档进行配置。 + +如何利用单机的所有GPU或所有CPU核心 +================================== + +PaddlePaddle的单机进程 :code:`paddle train` 可以充分利用一台计算机上所有的GPU资 +源或者CPU。 + +如果要使用机器上多块GPU，使用如下命令即可\: + +.. code-block:: bash + + paddle train --use_gpu=true --trainer_count=4 # use 4 gpu card, 0, 1, 2, 3 + +如果要使用机器上多块CPU, 使用如下命令即可\: + +.. code-block:: bash + + paddle train --trainer_config=4 # use 4 cpu cores. + +对于其他设置GPU的选择情况，例如选择第0、2号GPU显卡，则可以使用 :code:`CUDA_VISIBLE_DEVICES` 环境变量来选择部分的显卡。 具体可以参考连接`masking-gpus`_ 。 可以使用的命令为 + +.. code-block:: bash + + env CUDA_VISIBLE_DEVICES=0,2 paddle train --use_gpu=true --trainer_config=2 + +如何利用多台机器的计算资源训练神经网络 +====================================== + +PaddlePaddle多机使用的经典方法是通过 :code:`Parameter Server` 来对多机的 :code:`paddle train` 进行同步。 而多机训练神经网络，首先要讲数据切分到不同的机器上。 切分数据文件的方式在PaddlePaddle的开源实现中并没有提供工具包。 但是切分数据并不是一件非常复杂的事情，也不是神经网络实现的重点。 + +多机训练过程中，经典的拓扑结构如下\: + +.. graphviz:: pserver_topology.dot + +图中每个灰色方块是一台机器，在每个机器中，先去启动一个 :code:`paddle pserver` 进程，并确定整体的端口号。可能的参数是\: + +.. code-block:: bash + + paddle pserver --port=5000 --num_gradient_servers=4 --nics='eth0' + +这里说明系统的 :code:`paddle pserver` 的起始端口是 :code:`5000` ，并且有四个训练进程(:code:`gradient_servers`，Paddle同时将 :code:`paddle train` 进程称作 :code:`GradientServer` 。因为其为负责提供Gradient的进程)。 而对于训练进程的话，则需要在 :code:`paddle pserver` 启动之后，再在各个节点上运行如下命令\: + +.. code-block:: bash + + paddle train --port=5000 --pservers=192.168.100.101,192.168.100.102,192.168.100.103,192.168.100.104 --config=... + +对于简单的多机协同使用上述方式即可。同时，pserver/train 通常在高级情况下，还有两个参数需要设置，他们是 + +* --ports_num\: 一个 pserver进程共绑定多少个端口用来做稠密更新。默认是1 +* --ports_num_for_sparse\: 一个pserver进程共绑定多少端口用来做稀疏更新，默认是0 + +使用手工指定端口数量，是因为Paddle的网络通信中，使用了 :code:`int32` 作为消息长度，比较容易在大模型下溢出。所以，在 :code:`paddle pserver` 进程中可以启动多个子线程去接受 trainer 的数据，这样单个子线程的长度就不会溢出了。但是这个值不可以调的过大，因为增加这个值，还是对性能，尤其是内存占用有一定的开销的，另外稀疏更新的端口如果太大的话，很容易某一个参数服务器没有分配到任何参数。 + +详细的说明可以参考，使用 `集群训练Paddle`_ 。 + + +.. _PyDataProvider: ../ui/data_provider/pydataprovider2.html +.. _settings: ../../doc/ui/api/trainer_config_helpers/optimizers.html#settings +.. _mixed_layer: ../../doc/ui/api/trainer_config_helpers/layers.html#mixed-layer +.. _masking-gpu: http://www.acceleware.com/blog/cudavisibledevices-masking-gpus +.. _集群训练Paddle: ../cluster/index.html diff --git a/doc_cn/conf.py.in b/doc_cn/conf.py.in index 391f7981eab80..93242ace40600 100644 --- a/doc_cn/conf.py.in +++ b/doc_cn/conf.py.in @@ -47,6 +47,7 @@ extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', + 'sphinx.ext.graphviz' ] table_styling_embed_css = True diff --git a/doc_cn/demo/quick_start/index.md b/doc_cn/demo/quick_start/index.md index aa6b66ca8c024..4d9b24ba851a7 100644 --- a/doc_cn/demo/quick_start/index.md +++ b/doc_cn/demo/quick_start/index.md @@ -134,9 +134,8 @@ define_py_data_sources2(train_list='data/train.list', * obj="process": 指定生成数据的函数 * args={"dictionary": word_dict}: 额外的参数，这里指定词典 -更详细用例请参考文档Python Use Case， -数据格式和详细文档请参考 -PyDataProviderWrapper。 +更详细数据格式和用例请参考 +PyDataProvider2。 ## 网络结构(Network Architecture) 本节我们将专注于网络结构的介绍。 diff --git a/doc_cn/faq/index.rst b/doc_cn/faq/index.rst new file mode 100644 index 0000000000000..3eb0e10ae2228 --- /dev/null +++ b/doc_cn/faq/index.rst @@ -0,0 +1,216 @@ +#################### +PaddlePaddle常见问题 +#################### + +.. contents:: + +1. 如何减少PaddlePaddle的内存占用 +--------------------------------- + +神经网络的训练本身是一个非常消耗内存和显存的工作。经常会消耗数十G的内存和数G的显存。 +PaddlePaddle的内存占用主要分为如下几个方面\: + +* DataProvider缓冲池内存 (只针对内存) +* 神经元激活内存 （针对内存和显存） +* 参数内存 (针对内存和显存) +* 其他内存杂项 + +这其中，其他内存杂项是指PaddlePaddle本身所用的一些内存，包括字符串分配，临时变量等等， +这些内存就不考虑如何缩减了。 + +其他的内存的减少方法依次为 + + +减少DataProvider缓冲池内存 +++++++++++++++++++++++++++ + +PyDataProvider使用的是异步加载，同时在内存里直接随即选取数据来做Shuffle。即 + +.. graphviz:: + + digraph { + rankdir=LR; + 数据文件 -> 内存池 -> PaddlePaddle训练 + } + +所以，减小这个内存池即可减小内存占用，同时也可以加速开始训练前数据载入的过程。但是，这 +个内存池实际上决定了shuffle的粒度。所以，如果将这个内存池减小，又要保证数据是随机的， +那么最好将数据文件在每次读取之前做一次shuffle。可能的代码为 + +.. literalinclude:: reduce_min_pool_size.py + +这样做可以极大的减少内存占用，并且可能会加速训练过程。 详细文档参考 `这里 +<../ui/data_provider/pydataprovider2.html#provider>`_ 。 + +神经元激活内存 +++++++++++++++ + +神经网络在训练的时候，会对每一个激活暂存一些数据，包括激活，參差等等。 +在反向传递的时候，这些数据会被用来更新参数。这些数据使用的内存主要和两个参数有关系， +一是batch size，另一个是每条序列(Sequence)长度。所以，其实也是和每个mini-batch中包含 +的时间步信息成正比。 + +所以，做法可以有两种。他们是 + +* 减小batch size。 即在网络配置中 :code:`settings(batch_size=1000)` 设置成一个小一些的值。但是batch size本身是神经网络的超参数，减小batch size可能会对训练结果产生影响。 +* 减小序列的长度，或者直接扔掉非常长的序列。比如，一个数据集大部分序列长度是100-200, + 但是突然有一个10000长的序列，就很容易导致内存超限。特别是在LSTM等RNN中。 + +参数内存 +++++++++ + +PaddlePaddle支持非常多的优化算法(Optimizer)，不同的优化算法需要使用不同大小的内存。 +例如如果使用 :code:`adadelta` 算法，则需要使用参数规模大约5倍的内存。 如果参数保存下来的 +文件为 :code:`100M`， 那么该优化算法至少需要 :code:`500M` 的内存。 + +可以考虑使用一些优化算法，例如 :code:`momentum`。 + +2. 如何加速PaddlePaddle的训练速度 +--------------------------------- + +PaddlePaddle是神经网络训练平台，加速PaddlePaddle训练有如下几个方面\： + +* 减少数据载入的耗时 +* 加速训练速度 +* 利用更多的计算资源 + +减少数据载入的耗时 +++++++++++++++++++ + +使用 :code:`pydataprovider`时，可以减少缓存池的大小，同时设置内存缓存功能，即可以极大的加速数据载入流程。 +:code:`DataProvider` 缓存池的减小，和之前减小通过减小缓存池来减小内存占用的原理一致。 + +.. literalinclude:: reduce_min_pool_size.py + +同时 :code:`@provider` 接口有一个 :code:`cache` 参数来控制缓存方法，将其设置成 :code:`CacheType.CACHE_PASS_IN_MEM` 的话，会将第一个 :code:`pass` (过完所有训练数据即为一个pass)生成的数据缓存在内存里，在之后的 :code:`pass` 中，不会再从 :code:`python` 端读取数据，而是直接从内存的缓存里读取数据。这也会极大减少数据读入的耗时。 + + +加速训练速度 +++++++++++++ + +PaddlePaddle支持Sparse的训练，sparse训练需要训练特征是 :code:`sparse_binary_vector` 、 :code:`sparse_vector` 、或者 :code:`integer_value` 的任一一种。同时，与这个训练数据交互的Layer，需要将其Parameter设置成 sparse 更新模式，即设置 :code:`sparse_update=True` + +这里使用简单的 :code:`word2vec` 训练语言模型距离，具体使用方法为\: + +使用一个词前两个词和后两个词，来预测这个中间的词。这个任务的DataProvider为\: + +.. literalinclude:: word2vec_dataprovider.py + +这个任务的配置为\: + +.. literalinclude:: word2vec_config.py + +更多关于sparse训练的内容请参考 `sparse训练的文档 `_ + +利用更多的计算资源 +++++++++++++++++++ + +利用更多的计算资源可以分为一下几个方式来进行\: + +* 单机CPU训练 + * 使用多线程训练。设置命令行参数 :code:`trainer_count`，即可以设置参与训练的线程数量。使用方法为 :code:`paddle train --trainer_count=4` +* 单机GPU训练 + * 使用显卡训练。设置命令行参数 :code:`use_gpu`。 使用方法为 :code:`paddle train --use_gpu=true` + * 使用多块显卡训练。设置命令行参数 :code:`use_gpu` 和 :code:`trainer_count`。使用 :code:`--use_gpu=True` 开启GPU训练，使用 :code:`trainer_count` 指定显卡数量。使用方法为 :code:`paddle train --use_gpu=true --trainer_count=4` +* 多机训练 + * 使用多机训练的方法也比较简单，需要先在每个节点启动 :code:`paddle pserver`，在使用 :code:`paddle train --pservers=192.168.100.1,192.168.100.2` 来指定每个pserver的ip地址 + * 具体的多机训练方法参考 `多机训练 `_ 文档。 + + +3. 遇到“非法指令”或者是“illegal instruction” +-------------------------------------------- + +paddle在进行计算的时候为了提升计算性能，使用了avx指令。部分老的cpu型号无法支持这样的指令。通常来说执行下grep avx /proc/cpuinfo看看是否有输出即可知道是否支持。（另：用此方法部分虚拟机可能检测到支持avx指令但是实际运行会挂掉，请当成是不支持，看下面的解决方案） + +解决办法是\: + +* 使用 NO_AVX的 `安装包 <../build_and_install/index.html>`_ 或者 `Docker image <../build_and_install/install/docker_install.html>`_ +* 或者，使用 :code:`-DWITH_AVX=OFF` 重新编译PaddlePaddle。 + + +4. 如何选择SGD算法的学习率 +-------------------------- + +在采用sgd/async_sgd进行训练时，一个重要的问题是选择正确的learning_rate。如果learning_rate太大，那么训练有可能不收敛，如果learning_rate太小，那么收敛可能很慢，导致训练时间过长。 + +通常做法是从一个比较大的learning_rate开始试，如果不收敛，那减少学习率10倍继续试验，直到训练收敛为止。那么如何判断训练不收敛呢？可以估计出如果模型采用不变的输出最小的cost0是多少。 + +如果训练过程的的cost明显高于这个常数输出的cost，那么我们可以判断为训练不收敛。举一个例子，假如我们是三分类问题，采用multi-class-cross-entropy作为cost，数据中0,1,2三类的比例为 :code:`0.2, 0.5, 0.3` , 那么常数输出所能达到的最小cost是 :code:`-(0.2*log(0.2)+0.5*log(0.5)+0.3*log(0.3))=1.03` 。如果训练一个pass（或者更早）后，cost还大于这个数，那么可以认为训练不收敛，应该降低学习率。 + + +5. 如何初始化参数 +----------------- + +默认情况下，PaddlePaddle使用均值0，标准差为 :math:`\frac{1}{\sqrt{d}}` 来初始化参数。其中 :math:`d` 为参数矩阵的宽度。这种初始化方式在一般情况下不会产生很差的结果。如果用户想要自定义初始化方式，PaddlePaddle目前提供两种参数初始化的方式\: + +* 高斯分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_mean=0.0, initial_std=1.0)` +* 均匀分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0)` + +比如设置一个全连接层的参数初始化方式和bias初始化方式，可以使用如下代码。 + +.. code-block:: python + + hidden = fc_layer(input=ipt, param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0), + bias_attr=ParamAttr(initial_mean=1.0, initial_std=0.0)) + +上述代码将bias全部初始化为1.0, 同时将参数初始化为 :code:`[1.0, -1.0]` 的均匀分布。 + +6. 如何共享参数 +--------------- + +PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字的参数，会共享参数。设置参数的名字，可以使用 :code:`ParamAttr(name="YOUR_PARAM_NAME")` 来设置。更方便的设置方式，是想要共享的参数使用同样的 :code:`ParamAttr` 对象。 + +简单的全连接网络，参数共享的配置示例为\: + +.. literalinclude:: ../../python/paddle/trainer_config_helpers/tests/configs/shared_fc.py + +这里 :code:`hidden_a` 和 :code:`hidden_b` 使用了同样的parameter和bias。并且softmax层的两个输入也使用了同样的参数 :code:`softmax_param`。 + +7. *-cp27mu-linux_x86_64.whl is not a supported wheel on this platform. +----------------------------------------------------------------------- + +出现这个问题的主要原因是，系统编译wheel包的时候，使用的 :code:`wheel` 包是最新的， +而系统中的 :code:`pip` 包比较老。具体的解决方法是，更新 :code:`pip` 包并重新编译PaddlePaddle。 +更新 :code:`pip` 包的方法是\: + +.. code-block:: bash + + pip install --upgrade pip + +8. python相关的单元测试都过不了 +-------------------------------- + +如果出现以下python相关的单元测试都过不了的情况： + +.. code-block:: bash + + 24 - test_PyDataProvider (Failed) + 26 - test_RecurrentGradientMachine (Failed) + 27 - test_NetworkCompare (Failed) + 28 - test_PyDataProvider2 (Failed) + 32 - test_Prediction (Failed) + 33 - test_Compare (Failed) + 34 - test_Trainer (Failed) + 35 - test_TrainerOnePass (Failed) + 36 - test_CompareTwoNets (Failed) + 37 - test_CompareTwoOpts (Failed) + 38 - test_CompareSparse (Failed) + 39 - test_recurrent_machine_generation (Failed) + 40 - test_PyDataProviderWrapper (Failed) + 41 - test_config_parser (Failed) + 42 - test_swig_api (Failed) + 43 - layers_test (Failed) + +并且查询PaddlePaddle单元测试的日志，提示： + +.. code-block:: bash + + paddle package is already in your PYTHONPATH. But unittest need a clean environment. + Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'. + +解决办法是：卸载paddle包 :code:`pip uninstall paddle`。 + +原因是：单元测试使用了一个旧版本的python包，而没有测试到代码中实际修改的python包。即单元测试需要一个干净的环境： + +* 如果paddle包已经在python的site-packages里面了，那么单元测试时使用的paddle包，就是site-packages里面的python包，而不是源码目录里 :code:`/python` 目录下的python包。 +* 即便设置了 :code:`PYTHONPATH` 到 :code:`/python` 也没用，因为python的搜索路径是优先已经安装的python包。 \ No newline at end of file diff --git a/doc_cn/faq/reduce_min_pool_size.py b/doc_cn/faq/reduce_min_pool_size.py new file mode 100644 index 0000000000000..5715397cc11e1 --- /dev/null +++ b/doc_cn/faq/reduce_min_pool_size.py @@ -0,0 +1,6 @@ +@provider(min_pool_size=0, ...) +def process(settings, filename): + os.system('shuf %s > %s.shuf' % (filename, filename)) # shuffle before. + with open('%s.shuf' % filename, 'r') as f: + for line in f: + yield get_sample_from_line(line) diff --git a/doc_cn/faq/word2vec_config.py b/doc_cn/faq/word2vec_config.py new file mode 100644 index 0000000000000..866b40c3d4c96 --- /dev/null +++ b/doc_cn/faq/word2vec_config.py @@ -0,0 +1,12 @@ +... # the settings and define data provider is omitted. +DICT_DIM = 3000 # dictionary dimension. +word_ids = data_layer('word_ids', size=DICT_DIM) + +emb = embedding_layer( + input=word_ids, size=256, param_attr=ParamAttr(sparse_update=True)) +emb_sum = pooling_layer(input=emb, pooling_type=SumPooling()) +predict = fc_layer(input=emb_sum, size=DICT_DIM, act=Softmax()) +outputs( + classification_cost( + input=predict, label=data_layer( + 'label', size=DICT_DIM))) diff --git a/doc_cn/faq/word2vec_dataprovider.py b/doc_cn/faq/word2vec_dataprovider.py new file mode 100644 index 0000000000000..ec2753a7d01d7 --- /dev/null +++ b/doc_cn/faq/word2vec_dataprovider.py @@ -0,0 +1,10 @@ +DICT_DIM = 3000 + + +@provider(input_types=[integer_sequence(DICT_DIM), integer_value(DICT_DIM)]) +def process(settings, filename): + with open(filename) as f: + # yield word ids to predict inner word id + # such as [28, 29, 10, 4], 4 + # It means the sentance is 28, 29, 4, 10, 4. + yield read_next_from_file(f) diff --git a/doc_cn/howto/how_to_write_docs/index.rst b/doc_cn/howto/how_to_write_docs/index.rst new file mode 100644 index 0000000000000..869ef747f9f88 --- /dev/null +++ b/doc_cn/howto/how_to_write_docs/index.rst @@ -0,0 +1,63 @@ +############################### +如何贡献/修改PaddlePaddle的文档 +############################### + +PaddlePaddle的文档使用 `cmake`_ 驱动 `sphinx`_ 生成。公有两个文档，:code:`doc` 和 :code:`doc_cn` 。这两者会在 `cmake`_ 中进行编译，生成后的文档会存储在服务器的 :code:`doc` 和 :code:`doc_cn` 两个目录下。 + +下面分几个部分介绍一下PaddlePaddle文档的贡献方法。 + +如何书写PaddlePaddle的文档 +========================== + +TBD + +如何构建PaddlePaddle的文档 +========================== + +构建PaddlePaddle文档，需要使用构建Paddle的全部环境。准备这个环境相对来说比较复杂，所以本文档提供两种方式构建PaddlePaddle的文档，即 + +* 使用Docker构建PaddlePaddle的文档 +* 直接构建PaddlePaddle的文档。 + +并且，我们推荐使用Docker来构建PaddlePaddle的文档。 + + +使用Docker构建PaddlePaddle的文档 +-------------------------------- + +使用Docker构建PaddlePaddle的文档，首先要求在系统里安装好Docker工具包。安装Docker请参考 `Docker的官网 `_ 。 + +安装好Docker之后可以使用源码目录下的脚本构建文档，即 + +.. code-block:: bash + + cd TO_YOUR_PADDLE_CLONE_PATH + cd paddle/scripts/tools/build_docs + bash build_docs.sh + +执行完这个脚本后，该目录下会生成两个目录，分别是\: + +* doc 目录，英文文档地址 +* doc_cn 目录，中文文档地址 + +打开浏览器访问对应目录下的index.html即可访问本地文档。 + +.. code-block:: bash + + open doc_cn/index.html + + +直接构建PaddlePaddle的文档 +-------------------------- + +TBD + + +如何更新www.paddlepaddle.org文档 +================================ + +TBD + + +.. _cmake: https://cmake.org/ +.. _sphinx: http://www.sphinx-doc.org/en/1.4.8/ \ No newline at end of file diff --git a/doc_cn/index.rst b/doc_cn/index.rst index 6cf5588b5b34f..f1398206fddff 100644 --- a/doc_cn/index.rst +++ b/doc_cn/index.rst @@ -3,7 +3,9 @@ PaddlePaddle文档 使用指南 -------- +* `介绍 `_ * `快速入门 `_ +* `基本使用概念 `_ * `编译与安装 `_ * `用户接口 `_ * `使用示例 `_ @@ -13,7 +15,17 @@ PaddlePaddle文档 开发指南 -------- * `新写Layer <../doc/dev/new_layer/index.html>`_ +* `如何贡献文档 `_ 算法教程 -------- -* `RNN配置 <../doc/algorithm/rnn/rnn.html>`_ + +* `Recurrent Group教程 `_ +* `单层RNN示例 <../doc/algorithm/rnn/rnn.html>`_ +* `双层RNN示例 `_ +* `支持双层序列作为输入的Layer `_ + +常见问题 +-------- + +* `常见问题 `_ diff --git a/doc_cn/introduction/index.md b/doc_cn/introduction/index.md new file mode 100644 index 0000000000000..164cb7d4943df --- /dev/null +++ b/doc_cn/introduction/index.md @@ -0,0 +1,105 @@ +# 简介 + +PaddlePaddle 是起源于百度的开源深度学习平台。它是简单易用的：你可以通过简单的十数行配置搭建经典的神经网络模型；它也是高效强大的：PaddlePaddle可以支撑复杂集群环境下超大模型的训练，令你受益于深度学习的前沿成果。在百度内部，已经有大量产品线使用了基于PaddlePaddle的深度学习技术。 + +这份简短的介绍将像你展示如何利用PaddlePaddle解决一个经典的学习问题。 + +## 1. 一个经典的任务 + +让我们从一个基础问题开始：单变量的线性回归。问题假定观测到了一批二维空间上的点`(x, y) `，并且已知 `x` 和 `y` 之间存在着某种线性关系，我们的目标是通过观测数据还原这个线性关系。作为一个简单基础的模型，线性回归却有着广泛的应用场景。比如可以想象一个资产定价的简化场景，其中 `x` 对应于房屋的大小，`y` 对应于房屋价格。我们可以通过观察市场上房屋的情况获得二者之间的关系，从而为新房屋的定价提供参考。 + + +## 2. 准备数据 + +假设变量 `X` 和 `Y` 的真实关系为： `Y = 2X + 0.3`，这里展示如何使用观测数据还原这一线性关系。如下Python代码将随机产生2000个观测点，它们将被用作PaddlePaddle的输入。产生PaddlePaddle的输入数据和写一段普通的Python脚本几乎一样，你唯一需要增加的就是定义输入数据的类型。 + +```python +# -*- coding:utf-8 -*- +# dataprovider.py +from paddle.trainer.PyDataProvider2 import * +import random + +# 定义输入数据的类型: 2个浮点数 +@provider(input_types=[dense_vector(1), dense_vector(1)],use_seq=False) +def process(settings, input_file): + for i in xrange(2000): + x = random.random() + yield [x], [2*x+0.3] +``` + +## 3. 训练模型 + +为了还原 `Y = 2X + 0.3`，我们先从一条随机的直线 `Y' = wX + b` 开始，然后利用观测数据调整 `w` 和 `b` 使得 `Y'` 和 `Y` 的差距不断减小，最终趋于相同。这个过程就是模型的训练过程，而 `w` 和 `b` 就是模型的参数，即我们的训练目标。 + +在PaddlePaddle里，该模型的网络配置如下。 + +```python +# -*- coding:utf-8 -*- +# trainer_config.py +from paddle.trainer_config_helpers import * + +# 1. 定义数据来源，调用上面的process函数获得观测数据 +data_file = 'empty.list' +with open(data_file, 'w') as f: f.writelines(' ') +define_py_data_sources2(train_list=data_file, test_list=None, + module='dataprovider', obj='process',args={}) + +# 2. 学习算法。控制如何改变模型参数 w 和 b +settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) + +# 3. 神经网络配置 +x = data_layer(name='x', size=1) +y = data_layer(name='y', size=1) +# 线性计算单元: y_predict = wx + b +y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) +# 损失计算，度量 y_predict 和真实 y 之间的差距 +cost = regression_cost(input=y_predict, label=y) +outputs(cost) +``` +这段简短的配置展示了PaddlePaddle的基本用法： + +- 首先，第一部分定义了数据输入。一般情况下，PaddlePaddle先从一个文件列表里获得数据文件地址，然后交给用户自定义的函数（例如上面的`process`函数）进行读入和预处理从而得到真实输入。本文中由于输入数据是随机生成的不需要读输入文件，所以放一个空列表（`empty.list`）即可。 + +- 第二部分主要是选择学习算法，它定义了模型参数如何改变。PaddlePaddle提供了很多优秀的学习算法，但这里使用一个简单的基于momentum的算法就足够了，它每次读取12个数据进行计算和模型更新。 + +- 最后一部分是神经网络的配置。由于PaddlePaddle已经实现了丰富的网络单元（Layer），所以很多时候你需要做的只是声明正确的网络单元并把它们拼接起来。这里使用了三种网络单元： + - **数据层**：数据层 `data_layer` 是神经网络的入口，它读入数据并将它们传输到下游的其它单元。这里数据层有两个，分别对应于变量 `X` 和 `Y`。 + - **全连接层**：全连接层 `fc_layer` 是基础的计算单元，这里利用它建模变量之间的线性关系。计算单元是神经网络的核心，PaddlePaddle支持大量的计算单元和任意深度的网络连接，从而可以挖掘复杂的数据关系。 + - **回归损失层**：回归损失层 `regression_cost`是众多损失函数层的一种，它们在训练过程作为网络的出口，用来计算模型的表现，并指导模型参数的改变。 + +这样定义了网络结构并保存为`trainer_config.py`之后，运行训练命令即可： + ``` + paddle train --config=trainer_config.py --save_dir=./output --num_passes=30 + ``` + +PaddlePaddle将在观测数据集上迭代训练30轮，并将每轮的模型结果存放在 `./output` 路径下。从输出日志可以看到，随着轮数增加损失函数的输出在不断的减小，这意味着模型在不断的改进，直到逼近真实解：` Y = 2X + 0.3 ` + +## 4. 模型检验 + +训练完成后，我们希望能够检验模型的好坏。一种常用的做法是用模型对另外一组数据进行预测，然后评价预测的效果。但在这个例子中，由于已经知道了真实答案，我们可以直接观察模型的参数是否符合预期来进行检验。 + +PaddlePaddle将每个模型参数作为一个numpy数组单独存为一个文件，所以可以利用如下方法读取模型的参数。 + +```python +import numpy as np +import os + +def load(file_name): + with open(file_name, 'rb') as f: + f.read(16) # skip header for float type. + return np.fromfile(f, dtype=np.float32) + +print 'w=%.6f, b=%.6f' % (load('output/pass-00029/w'), load('output/pass-00029/b')) +# w=1.999743, b=0.300137 +``` +
![](./parameters.png)
+ +从图中可以看到，虽然 `w` 和 `b` 都使用随机值初始化，但在起初的几轮训练中它们都在快速逼近真实值，并且后续仍在不断改进，使得最终得到的模型几乎与真实模型重合。 + +这样，我们就完成了对单变量线性回归问题的解决：将数据输入PaddlePaddle，训练模型，最后验证结果。 + +## 5. 推荐后续阅读 + +- 安装/编译：PaddlePaddle的安装与编译文档。 +- 快速入门 ：使用商品评论分类任务，系统性的介绍如何一步步改进，最终得到产品级的深度模型。 +- 示例：各种实用案例，涵盖图像、文本、推荐等多个领域。 diff --git a/doc_cn/introduction/parameters.png b/doc_cn/introduction/parameters.png new file mode 100644 index 0000000000000..2ec67480951e2 Binary files /dev/null and b/doc_cn/introduction/parameters.png differ diff --git a/doc_cn/ui/data_provider/mnist_config.py b/doc_cn/ui/data_provider/mnist_config.py index 7ba344338c374..39becff03b08f 100644 --- a/doc_cn/ui/data_provider/mnist_config.py +++ b/doc_cn/ui/data_provider/mnist_config.py @@ -1,8 +1,9 @@ from paddle.trainer_config_helpers import * -define_py_data_sources2(train_list='train.list', - test_list=None, - module='mnist_provider', - obj='process') +define_py_data_sources2( + train_list='train.list', + test_list=None, + module='mnist_provider', + obj='process') img = data_layer(name='pixel', size=784) label = data_layer(name='label', size=10) diff --git a/doc_cn/ui/data_provider/mnist_provider.dict.py b/doc_cn/ui/data_provider/mnist_provider.dict.py index 4eab5b1fd3b50..2ba0b126a0d62 100644 --- a/doc_cn/ui/data_provider/mnist_provider.dict.py +++ b/doc_cn/ui/data_provider/mnist_provider.dict.py @@ -2,10 +2,9 @@ # Define a py data provider -@provider(input_types=[ - dense_vector(28 * 28), - integer_value(10) -]) +@provider( + input_types={'pixel': dense_vector(28 * 28), + 'label': integer_value(10)}) def process(settings, filename): # settings is not used currently. f = open(filename, 'r') # open one of training file @@ -20,6 +19,6 @@ def process(settings, filename): # settings is not used currently. pixels_float.append(float(each_pixel_str)) # give data to paddle. - yield { "pixel": pixels_float, 'label': int(label) } + yield {"pixel": pixels_float, 'label': int(label)} f.close() # close file diff --git a/doc_cn/ui/data_provider/mnist_provider.py b/doc_cn/ui/data_provider/mnist_provider.py index 92f1915c10725..8b828641d5573 100644 --- a/doc_cn/ui/data_provider/mnist_provider.py +++ b/doc_cn/ui/data_provider/mnist_provider.py @@ -2,10 +2,7 @@ # Define a py data provider -@provider(input_types=[ - dense_vector(28 * 28), - integer_value(10) -]) +@provider(input_types=[dense_vector(28 * 28), integer_value(10)]) def process(settings, filename): # settings is not used currently. f = open(filename, 'r') # open one of training file diff --git a/doc_cn/ui/data_provider/pydataprovider2.rst b/doc_cn/ui/data_provider/pydataprovider2.rst index 9e1d8c531f5ba..80b40084d8f50 100644 --- a/doc_cn/ui/data_provider/pydataprovider2.rst +++ b/doc_cn/ui/data_provider/pydataprovider2.rst @@ -141,8 +141,6 @@ DataProvider创建的时候执行。这个初始化函数具有如下参数: 是一个batch size，但是有时为了计算均衡性，可以将一条数据设置成多个batch size * cache 是数据缓存的策略，参考 `cache`_ * init_hook 是初始化时调用的函数，参考 `init_hook`_ -* use_dynamic_order 如果是true的话，可以返回一个dict，key是data_layer的名字，value是特征值。同时，也可以 - 返回一个list或者tuple。如果是false的话，只能够返回list或者tuple * check 设置成true的话，会根据input_types检查数据的合法性。 * check_fail_continue 如果设置成true的话，即使在check中数据不合法，也会扔到这条数据，继续训练。 如果 check是false的话，没有作用。 diff --git a/doc_cn/ui/data_provider/sentimental_config.py b/doc_cn/ui/data_provider/sentimental_config.py index 051f75e32b5c0..7ce71608a2372 100644 --- a/doc_cn/ui/data_provider/sentimental_config.py +++ b/doc_cn/ui/data_provider/sentimental_config.py @@ -3,9 +3,12 @@ dictionary = dict() ... # read dictionary from outside -define_py_data_sources2(train_list='train.list', test_list=None, - module='sentimental_provider', obj='process', - # above codes same as mnist sample. - args={ # pass to provider. - 'dictionary': dictionary - }) +define_py_data_sources2( + train_list='train.list', + test_list=None, + module='sentimental_provider', + obj='process', + # above codes same as mnist sample. + args={ # pass to provider. + 'dictionary': dictionary + }) diff --git a/doc_cn/ui/data_provider/sentimental_provider.py b/doc_cn/ui/data_provider/sentimental_provider.py index bda37d7722a0b..0fb0bb88e95a2 100644 --- a/doc_cn/ui/data_provider/sentimental_provider.py +++ b/doc_cn/ui/data_provider/sentimental_provider.py @@ -12,7 +12,8 @@ def on_init(settings, dictionary, **kwargs): # The text is a sequence of integer values, and each value is a word id. # The whole sequence is the sentences that we want to predict its # sentimental. - integer_value(len(dictionary), seq_type=SequenceType), # text input + integer_value( + len(dictionary), seq_type=SequenceType), # text input # label positive/negative integer_value(2) diff --git a/paddle/.common_test_util.sh b/paddle/.common_test_util.sh index dec22e45619fb..dc15250615908 100644 --- a/paddle/.common_test_util.sh +++ b/paddle/.common_test_util.sh @@ -117,4 +117,4 @@ set_port() fi done -} \ No newline at end of file +} diff --git a/paddle/.set_python_path.sh b/paddle/.set_python_path.sh index f7019b27f8f02..657fdf65e92c9 100755 --- a/paddle/.set_python_path.sh +++ b/paddle/.set_python_path.sh @@ -33,7 +33,7 @@ if ! python -c "import paddle" >/dev/null 2>/dev/null; then esac done shift $(($OPTIND - 1)) - export PYTHONPATH=$PYPATH + export PYTHONPATH=$PYPATH:$PYTHONPATH $@ else echo "paddle package is already in your PYTHONPATH. But unittest need a clean environment." diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index cae0f64400a7e..fb3af8ea92fee 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -17,5 +17,3 @@ endif() if(WITH_SWIG_PY) add_subdirectory(api) endif() - - diff --git a/paddle/api/Arguments.cpp b/paddle/api/Arguments.cpp index 8f73e7626042c..6f51d55120069 100644 --- a/paddle/api/Arguments.cpp +++ b/paddle/api/Arguments.cpp @@ -14,27 +14,10 @@ limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include "paddle/parameter/Argument.h" -struct ArgumentsPrivate { - std::vector outputs; - - inline paddle::Argument& getArg(size_t idx) throw(RangeError) { - if (idx < outputs.size()) { - return outputs[idx]; - } else { - RangeError e; - throw e; - } - } - - template - std::shared_ptr& cast(void* rawPtr) const { - return *(std::shared_ptr*)(rawPtr); - } -}; - size_t Arguments::getSlotNum() const { return m->outputs.size(); } Arguments* Arguments::createArguments(size_t slotNum) { diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index fe0da763514a6..9b2d122a09ada 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -40,6 +40,8 @@ configure_file( generate_python_api(python_swig_sources) +file(GLOB PY_PADDLE_PYTHON_FILES ${PROJ_ROOT}/paddle/py_paddle/*.py) + # TODO(yuyang18) : make wheel name calculated by cmake add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel @@ -55,6 +57,7 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp paddle_trainer paddle_api paddle_cuda + ${PY_PADDLE_PYTHON_FILES} ) install(DIRECTORY ${PROJ_ROOT}/paddle/dist/ diff --git a/paddle/api/ConfigParser.cpp b/paddle/api/ConfigParser.cpp index c5ee784a0bda0..25d94f5a6a125 100644 --- a/paddle/api/ConfigParser.cpp +++ b/paddle/api/ConfigParser.cpp @@ -14,17 +14,9 @@ limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include "paddle/trainer/Trainer.h" -struct TrainerConfigPrivate { - std::shared_ptr conf; - TrainerConfigPrivate() : conf(std::make_shared()) {} -}; - -struct ModelConfigPrivate { - std::shared_ptr conf; -}; - struct ParameterConfigPrivate { paddle::ParameterPtr parameter; paddle::ParameterConfig config; @@ -39,19 +31,6 @@ struct ParameterConfigPrivate { } }; -struct OptimizationConfigPrivate { - std::shared_ptr trainer_config; - paddle::OptimizationConfig config; - - paddle::OptimizationConfig& getConfig() { - if (trainer_config != nullptr) { - return *trainer_config->mutable_opt_config(); - } else { - return config; - } - } -}; - TrainerConfig::TrainerConfig() : m(new TrainerConfigPrivate()) {} TrainerConfig::~TrainerConfig() { delete m; } @@ -59,10 +38,19 @@ TrainerConfig::~TrainerConfig() { delete m; } TrainerConfig* TrainerConfig::createFromTrainerConfigFile( const std::string& confPath) { LOG(INFO) << "load trainer config from " << confPath; - paddle::TrainerConfigHelper helper(confPath); - //! TODO(yuyang18): Make TrainerConfigPrivate to TrainerConfigHelper + auto conf = std::make_shared(confPath); auto retv = new TrainerConfig(); - *retv->m->conf = helper.getConfig(); + retv->m->conf = conf; + return retv; +} + +TrainerConfig* TrainerConfig::createFromProtoString( + const std::string& str) { + auto retv = new TrainerConfig(); + paddle::TrainerConfig trainerConfigProto; + auto conf = std::make_shared(trainerConfigProto); + CHECK(conf->getMutableConfig().ParseFromString(str)); + retv->m->conf = conf; return retv; } @@ -76,10 +64,6 @@ ModelConfig* TrainerConfig::getModelConfig() const { return retv; } -void* ModelConfig::getPaddleModelConfig() const { - return m->conf->mutable_model_config(); -} - ParameterConfig::ParameterConfig() : m(new ParameterConfigPrivate()) {} ParameterConfig::~ParameterConfig() { @@ -132,8 +116,6 @@ OptimizationConfig* TrainerConfig::getOptimizationConfig() const { return opt_config; } -void* OptimizationConfig::getRawPtr() { return &m->getConfig(); } - OptimizationConfig* OptimizationConfig::createFromProtoString( const std::string& str) { auto conf = new OptimizationConfig(); diff --git a/paddle/api/GradientMachine.cpp b/paddle/api/GradientMachine.cpp index 6f1d63575a80f..bef499c67858b 100644 --- a/paddle/api/GradientMachine.cpp +++ b/paddle/api/GradientMachine.cpp @@ -14,30 +14,22 @@ limitations under the License. */ #include "PaddleAPI.h" -#include "paddle/gserver/gradientmachines/GradientMachine.h" +#include "PaddleAPIPrivate.h" + #include "paddle/gserver/gradientmachines/NeuralNetwork.h" #include "Internal.h" std::vector GradientMachine::defaultParamTypes = { PARAMETER_VALUE, PARAMETER_GRADIENT, PARAMETER_MOMENTUM}; -struct GradientMachinePrivate { - std::shared_ptr machine; - - template - inline T& cast(void* ptr) { - return *(T*)(ptr); - } -}; - GradientMachine::GradientMachine() : m(new GradientMachinePrivate()) {} GradientMachine::~GradientMachine() { delete m; } GradientMachine* GradientMachine::createFromPaddleModelPtr( - void* confPtr, GradientMatchineCreateMode mode, + const void* confPtr, GradientMatchineCreateMode mode, const std::vector& types) { - auto& conf = *(paddle::ModelConfig*)(confPtr); + auto& conf = *(const paddle::ModelConfig*)(confPtr); std::vector realTypes; staticCastVector(&realTypes, types); auto machineRawPtr = paddle::GradientMachine::create(conf, mode, realTypes); @@ -66,7 +58,7 @@ GradientMachine* GradientMachine::createByConfigProtoStr( GradientMachine* GradientMachine::createByModelConfig( ModelConfig* conf, GradientMatchineCreateMode mode, const std::vector& types) { - auto confPtr = (paddle::ModelConfig*)conf->getPaddleModelConfig(); + auto confPtr = &conf->m->conf->getModelConfig(); return GradientMachine::createFromPaddleModelPtr(confPtr, mode, types); } diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index b3140617af188..cf790f2f8ef1d 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -446,7 +446,6 @@ struct OptimizationConfigPrivate; class OptimizationConfig { DISABLE_COPY_AND_ASSIGN(OptimizationConfig); OptimizationConfig(); - void* getRawPtr(); public: static OptimizationConfig* createFromProtoString(const std::string& str); @@ -462,6 +461,7 @@ class OptimizationConfig { friend class TrainerConfig; friend class ParameterOptimizer; + friend class Trainer; }; struct ParameterPrivate; @@ -515,8 +515,6 @@ class ModelConfig { virtual ~ModelConfig(); private: - void* getPaddleModelConfig() const; - ModelConfigPrivate* m; friend class TrainerConfig; friend struct TrainerConfigPrivate; @@ -539,6 +537,7 @@ class TrainerConfig { static TrainerConfig* createFromTrainerConfigFile( const std::string& configPath); + static TrainerConfig* createFromProtoString(const std::string& str); ModelConfig* getModelConfig() const; @@ -546,6 +545,7 @@ class TrainerConfig { private: TrainerConfigPrivate* m; + friend class Trainer; }; /** @@ -700,11 +700,12 @@ class GradientMachine { GradientMachinePrivate* m; static GradientMachine* createFromPaddleModelPtr( - void* confPtr, GradientMatchineCreateMode mode, + const void* confPtr, GradientMatchineCreateMode mode, const std::vector& types); // Not to use c++ 11 init-list, so we use static var as function default arg. static std::vector defaultParamTypes; + friend class Trainer; }; struct TrainerPrivate; @@ -712,6 +713,7 @@ class Trainer { private: TrainerPrivate* m; Trainer(); + Trainer(TrainerConfig* optConfig, GradientMachine* gm); DISABLE_COPY_AND_ASSIGN(Trainer); public: @@ -720,38 +722,42 @@ class Trainer { /// Create A Trainer By TrainerConfig. using paddle command line. static Trainer* createByCommandLine() throw(IOError); - /// Start Train. + static Trainer* create(TrainerConfig* optConfig, GradientMachine* gm) + throw(IOError); + + /// Start training void startTrain(); + + /// Finish training void finishTrain(); - /// Start Pass. + /// Start a pass. void startTrainPass(); - void finishTrainPass(); - void setBatchSize(size_t batchSize); + /// Finish a pass + void finishTrainPass(); /** * Train one batch, * - * @param batchSize -1 wiil use command line or batch size set before, - * otherwise use this batchSize for train. - * * @return true if all batch finished. */ - bool trainOneBatch(size_t batchSize = -1UL); + bool trainOneBatch(size_t batchSize); - bool prepareBatchData(size_t batchSize = -1UL); + void trainOneDataBatch(size_t batchSize, const Arguments& args); - void finishTrainOneBatch(); + void startTestPeriod(); + void testOneDataBatch(size_t batchSize, const Arguments& args); + void finishTestPeriod(); - void forwardOneBatch() throw(UnsupportError); + void forwardOneBatch(size_t batchSize); - Arguments* getNetworkOutput(); + Arguments* getForwardOutput(); Matrix* getLayerOutput(const std::string& layerName); }; -/// The N-Best results generated from one input sequence. +/// the N-Best results generated from one input sequence. class ISequenceResults { public: virtual ~ISequenceResults(); diff --git a/paddle/api/PaddleAPIPrivate.h b/paddle/api/PaddleAPIPrivate.h new file mode 100644 index 0000000000000..5ffeff6a9726c --- /dev/null +++ b/paddle/api/PaddleAPIPrivate.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/gserver/gradientmachines/GradientMachine.h" +#include "paddle/trainer/TrainerConfigHelper.h" + +#pragma once + +struct GradientMachinePrivate { + std::shared_ptr machine; + + template + inline T& cast(void* ptr) { + return *(T*)(ptr); + } +}; + +struct OptimizationConfigPrivate { + std::shared_ptr trainer_config; + paddle::OptimizationConfig config; + + const paddle::OptimizationConfig& getConfig() { + if (trainer_config != nullptr) { + return trainer_config->getOptConfig(); + } else { + return config; + } + } +}; + +struct TrainerConfigPrivate { + std::shared_ptr conf; + TrainerConfigPrivate() {} +}; + +struct ModelConfigPrivate { + std::shared_ptr conf; +}; + +struct ArgumentsPrivate { + std::vector outputs; + + inline paddle::Argument& getArg(size_t idx) throw(RangeError) { + if (idx < outputs.size()) { + return outputs[idx]; + } else { + RangeError e; + throw e; + } + } + + template + std::shared_ptr& cast(void* rawPtr) const { + return *(std::shared_ptr*)(rawPtr); + } +}; diff --git a/paddle/api/ParameterOptimizer.cpp b/paddle/api/ParameterOptimizer.cpp index e087defc6043c..b13761ab0900d 100644 --- a/paddle/api/ParameterOptimizer.cpp +++ b/paddle/api/ParameterOptimizer.cpp @@ -14,6 +14,7 @@ limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include "paddle/parameter/ParameterOptimizer.h" #include "Internal.h" #include @@ -60,10 +61,9 @@ ParameterOptimizer::~ParameterOptimizer() { ParameterOptimizer* ParameterOptimizer::create(OptimizationConfig* config) { CHECK(config != nullptr); - auto opt_config_ptr = (paddle::OptimizationConfig*)config->getRawPtr(); auto retOptimizer = new ParameterOptimizer(); retOptimizer->m->optimizer.reset( - paddle::ParameterOptimizer::create(*opt_config_ptr, false)); + paddle::ParameterOptimizer::create(config->m->getConfig(), false)); return retOptimizer; } diff --git a/paddle/api/Trainer.cpp b/paddle/api/Trainer.cpp index 95b578c8db9fd..b61f36f740d47 100644 --- a/paddle/api/Trainer.cpp +++ b/paddle/api/Trainer.cpp @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include #include @@ -30,31 +31,17 @@ P_DECLARE_string(config); P_DECLARE_string(init_model_path); P_DECLARE_int32(start_pass); -struct TrainPassContext { - int64_t batchId; - int32_t batchSize; - real avgTestCost; - int64_t numAvgTests; - int passInnerId; - paddle::DataBatch data; - std::vector forwardOutput; -}; - struct TrainerPrivate : public paddle::Trainer { - void startTrain(); - void finishTrain(); - - void startTrainPass(); - void finishTrainPass(); - - bool _trainOneBatch(); - - bool _prepareBatchData(); - void _forwardOneBatch() throw(UnsupportError); - + bool _trainOneBatch(size_t batchSize); + bool forwardOneBatch(size_t batchSize); + void forwardOneDataBatch(const std::vector& inArgs); + void setBatchSize(size_t batchSize); + std::vector& getForwardOutput(); + + void startTestPeriod(); + void finishTestPeriod(); + void testOneDataBatch(const paddle::DataBatch& dataBatch); TrainerPrivate() : paddle::Trainer() {} - - TrainPassContext trainPassContext; }; Trainer::Trainer() : m(new TrainerPrivate()) { @@ -75,61 +62,76 @@ Trainer* Trainer::createByCommandLine() throw(IOError) { } } -void Trainer::startTrain() { m->startTrain(); } +Trainer::Trainer(TrainerConfig* config, GradientMachine* gm) + : m(new TrainerPrivate()) { + m->init(config->m->conf, /* testing= */false, gm ? gm->m->machine : nullptr); +} -void TrainerPrivate::startTrain() { - srand(this->config_->getConfig().start_pass() + 1); - this->dataProvider_->reset(); - this->trainerInternal_.getGradientMachine()->start(*config_, dataProvider_); +Trainer* Trainer::create(TrainerConfig* config, GradientMachine* gm) + throw(IOError) +{ + auto retv = new Trainer(config, gm); + if (retv->m->getConfig().IsInitialized()) { + return retv; + } else { + retv->m->getConfig().CheckInitialized(); + throw IOError(); + } } -void Trainer::finishTrain() { m->finishTrain(); } +void Trainer::startTrain() { m->startTrain(); } -void TrainerPrivate::finishTrain() { - this->trainerInternal_.getGradientMachine()->finish(); -} +void Trainer::finishTrain() { m->finishTrain(); } void Trainer::startTrainPass() { m->startTrainPass(); } -void TrainerPrivate::startTrainPass() { - this->stats_.reset(); - this->trainPassContext.batchId = 0; - this->trainPassContext.batchSize = this->config_->getOptConfig().batch_size(); - this->trainPassContext.avgTestCost = 0; - this->trainPassContext.numAvgTests = 0; - this->trainPassContext.passInnerId = 0; - this->trainerInternal_.getParameterUpdater()->startPass(); - this->evaluator_->start(); -} - void Trainer::finishTrainPass() { m->finishTrainPass(); } -void TrainerPrivate::finishTrainPass() { - this->trainerInternal_.getGradientMachine()->onPassEnd(); - this->trainerInternal_.getParameterUpdater()->finishPass(); - evaluator_->finish(); +void Trainer::trainOneDataBatch(size_t batchSize, const Arguments& inArgs) { + paddle::DataBatch dataBatch; + dataBatch.getStreams() = inArgs.m->outputs; + dataBatch.setSize(batchSize); + m->trainOneDataBatch(dataBatch); } -void Trainer::setBatchSize(size_t batchSize) { - this->m->trainPassContext.batchSize = batchSize; +bool Trainer::trainOneBatch(size_t batchSize) { + return m->_trainOneBatch(batchSize); } -bool Trainer::trainOneBatch(size_t batchSize) { - if (batchSize == -1UL) { - this->setBatchSize(batchSize); +bool TrainerPrivate::_trainOneBatch(size_t batchSize) { + paddle::DataBatch dataBatch; + CHECK(dataProvider_) << "data_provider is not specified"; + int num = dataProvider_->getNextBatch(batchSize, &dataBatch); + if (num == 0) { + return false; } - return m->_trainOneBatch(); + trainOneDataBatch(dataBatch); + return false; } -bool TrainerPrivate::_trainOneBatch() { - if (this->_prepareBatchData()) { - return true; +void TrainerPrivate::startTestPeriod() { + if (!tester_) { + createTester(); } - this->trainerInternal_.trainOneBatch(this->trainPassContext.batchId, - this->trainPassContext.data); - return false; + tester_->startTestPeriod(); +} + +void Trainer::startTestPeriod() { m->startTestPeriod(); } + +void TrainerPrivate::testOneDataBatch(const paddle::DataBatch& dataBatch) { + tester_->testOneDataBatch(dataBatch, &forwardOutput_); +} + +void Trainer::testOneDataBatch(size_t batchSize, const Arguments& args) { + paddle::DataBatch dataBatch; + dataBatch.getStreams() = args.m->outputs; + dataBatch.setSize(batchSize); + m->testOneDataBatch(dataBatch); } +void TrainerPrivate::finishTestPeriod() { tester_->finishTestPeriod(); } +void Trainer::finishTestPeriod() { m->finishTestPeriod(); } + Matrix* Trainer::getLayerOutput(const std::string& layerName) { auto nn = std::dynamic_pointer_cast( this->m->getGradientMachine()); @@ -138,46 +140,37 @@ Matrix* Trainer::getLayerOutput(const std::string& layerName) { return Matrix::createByPaddleMatrixPtr(&m); } -bool Trainer::prepareBatchData(size_t batchSize) { - if (batchSize != -1UL) { - this->setBatchSize(batchSize); +void Trainer::forwardOneBatch(size_t batchSize) { m->forwardOneBatch(batchSize); } + +bool TrainerPrivate::forwardOneBatch(size_t batchSize) { + CHECK(dataProvider_) << "data_provider is not specified"; + paddle::DataBatch dataBatch; + int num = dataProvider_->getNextBatch(batchSize, &dataBatch); + if (num == 0) { + return false; } - return this->m->_prepareBatchData(); -} -bool TrainerPrivate::_prepareBatchData() { - int num = dataProvider_->getNextBatch(this->trainPassContext.batchSize, - &this->trainPassContext.data); - return num == 0; + forwardOneDataBatch(dataBatch.getStreams()); + return true; } -void Trainer::finishTrainOneBatch() { ++m->trainPassContext.batchId; } +void TrainerPrivate::forwardOneDataBatch( + const std::vector& inArgs) { -void Trainer::forwardOneBatch() throw(UnsupportError) { m->_forwardOneBatch(); } - -void TrainerPrivate::_forwardOneBatch() throw(UnsupportError) { - auto& dataBatch = this->trainPassContext.data; - - int64_t actualBatchSize = dataBatch.getSize(); - if (actualBatchSize == 0) { - return; - } - - const std::vector& inArgs = dataBatch.getStreams(); - std::vector& outArgs = this->trainPassContext.forwardOutput; - outArgs.clear(); - paddle::PassType passType = - this->trainerInternal_.getParameterUpdater()->startBatch(actualBatchSize); + std::vector& outArgs = forwardOutput_; if (config_->getOptConfig().use_sparse_remote_updater()) { - this->trainerInternal_.getGradientMachine()->prefetch(inArgs); - this->trainerInternal_.getParameterUpdater()->getParametersRemote(); + trainerInternal_.getGradientMachine()->prefetch(inArgs); + trainerInternal_.getParameterUpdater()->getParametersRemote(); } - this->trainerInternal_.getGradientMachine()->forward( - inArgs, &outArgs, passType); + trainerInternal_.getGradientMachine()->forward( + inArgs, &outArgs, paddle::PASS_TEST); +} + +Arguments* Trainer::getForwardOutput() { + return Arguments::createByPaddleArgumentVector(&m->getForwardOutput()); } -Arguments* Trainer::getNetworkOutput() { - return Arguments::createByPaddleArgumentVector( - &m->trainPassContext.forwardOutput); +std::vector& TrainerPrivate::getForwardOutput() { + return forwardOutput_; } diff --git a/paddle/api/__init__.py b/paddle/api/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/paddle/api/__init__.py +++ b/paddle/api/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/api/paddle_api_config.py.in b/paddle/api/paddle_api_config.py.in index 6531e5ccb3dba..a2352250c31ef 100644 --- a/paddle/api/paddle_api_config.py.in +++ b/paddle/api/paddle_api_config.py.in @@ -1,6 +1,7 @@ PADDLE_BUILD_DIR="@CMAKE_CURRENT_BINARY_DIR@/../" WITH_GPU="@WITH_GPU@" PROTOBUF_LIB="@PROTOBUF_LIBRARY@" +ZLIB_LIB="@ZLIB_LIBRARIES@" CMAKE_THREAD_LIB="@CMAKE_THREAD_LIBS_INIT@" CMAKE_DL_LIBS="@CMAKE_DL_LIBS@" @@ -15,3 +16,4 @@ GFLAGS_LOCATION="@GFLAGS_LOCATION@" CBLAS_LIBRARIES="@CBLAS_LIBS@" CUDA_LIBRARIES="@CUDA_LIBRARIES@" +WITH_COVERALLS="@ON_COVERALLS@" diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index bc1afc5898e82..ebe00798e8b71 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -29,7 +29,10 @@ whole_start = "" whole_end = "" - LIB_DIRS = ["math", 'utils', 'parameter', "gserver", "api", "cuda", "pserver", "trainer"] + LIB_DIRS = [ + "math", 'utils', 'parameter', "gserver", "api", "cuda", "pserver", + "trainer" + ] PARENT_LIB_DIRS = ['proto'] class PaddleLDFlag(object): @@ -38,6 +41,7 @@ def __init__(self): self.paddle_build_dir = os.path.abspath(self.paddle_build_dir) self.with_gpu = PaddleLDFlag.cmake_bool(WITH_GPU) self.protolib = PROTOBUF_LIB + self.zlib = ZLIB_LIB self.thread = CMAKE_THREAD_LIB self.dl_libs = CMAKE_DL_LIBS self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON) @@ -47,25 +51,27 @@ def __init__(self): self.glog_libs = LIBGLOG_LIBRARY self.with_gflags = PaddleLDFlag.cmake_bool(WITH_GFLAGS) + self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS) self.gflags_libs = GFLAGS_LIBRARIES self.gflags_location = GFLAGS_LOCATION self.cblas_libs = CBLAS_LIBRARIES self.curt = CUDA_LIBRARIES def ldflag_str(self): - return " ".join([self.libs_dir_str(), - self.parent_dir_str(), - self.libs_str()]) + return " ".join( + [self.libs_dir_str(), self.parent_dir_str(), self.libs_str()]) def libs_dir_str(self): libdirs = LIB_DIRS - return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x), - libdirs)) + return " ".join( + map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x), + libdirs)) def parent_dir_str(self): libdirs = PARENT_LIB_DIRS - return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), - libdirs)) + return " ".join( + map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), + libdirs)) def libs_str(self): libs = [ @@ -82,6 +88,7 @@ def libs_str(self): "-lpaddle_cuda", "-lpaddle_api", self.normalize_flag(self.protolib), + self.normalize_flag(self.zlib), self.normalize_flag(self.thread), self.normalize_flag(self.dl_libs), self.normalize_flag(self.cblas_libs), @@ -95,6 +102,8 @@ def libs_str(self): libs.append(self.normalize_flag(self.gflags_libs)) if self.with_gpu: libs.append(self.normalize_flag(self.curt)) + if self.with_coverage: + libs.append("-fprofile-arcs") return " ".join(filter(lambda l: len(l) != 0, libs)) def normalize_flag(self, cmake_flag): @@ -108,10 +117,10 @@ def normalize_flag(self, cmake_flag): return cmake_flag elif cmake_flag.startswith("-l"): # normal link command return cmake_flag - elif cmake_flag in ["gflags-shared", - "gflags-static", - "gflags_nothreads-shared", - "gflags_nothreads-static"]: # special for gflags + elif cmake_flag in [ + "gflags-shared", "gflags-static", "gflags_nothreads-shared", + "gflags_nothreads-static" + ]: # special for gflags assert PaddleLDFlag.cmake_bool(self.gflags_location) return self.gflags_location elif len(cmake_flag) != 0: @@ -127,12 +136,22 @@ def cmake_bool(cmake_str): :type cmake_str: str :rtype: bool """ - if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith("-NOTFOUND"): + if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith( + "-NOTFOUND"): return False else: return True + def c_flag(self): + if self.with_coverage: + return ["-fprofile-arcs", "-ftest-coverage", "-O0", "-g"] + else: + return None except ImportError: + class PaddleLDFlag(object): def ldflag_str(self): pass + + def c_flag(self): + pass diff --git a/paddle/api/test/CMakeLists.txt b/paddle/api/test/CMakeLists.txt index c4c26e6c03fdf..08a0fe96a004d 100644 --- a/paddle/api/test/CMakeLists.txt +++ b/paddle/api/test/CMakeLists.txt @@ -1,2 +1,2 @@ add_test(NAME test_swig_api - COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh) \ No newline at end of file + COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh) diff --git a/paddle/api/test/run_tests.sh b/paddle/api/test/run_tests.sh index 1fc6fd5a8c185..a4814f98f89c2 100755 --- a/paddle/api/test/run_tests.sh +++ b/paddle/api/test/run_tests.sh @@ -30,7 +30,7 @@ source .test_env/bin/activate pip --timeout 600 install ../../dist/*.whl -test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py" +test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py" export PYTHONPATH=$PWD/../../../python/ diff --git a/paddle/api/test/testArguments.py b/paddle/api/test/testArguments.py index daedd2409effc..70fb169fd5c43 100644 --- a/paddle/api/test/testArguments.py +++ b/paddle/api/test/testArguments.py @@ -32,7 +32,7 @@ def test_load_arguments(self): iv = args.getSlotIds(0) assert isinstance(iv, swig_paddle.IVector) np_arr = iv.toNumpyArrayInplace() - self.assertEqual(np_arr.shape, (6,)) + self.assertEqual(np_arr.shape, (6, )) if __name__ == '__main__': diff --git a/paddle/api/test/testGradientMachine.py b/paddle/api/test/testGradientMachine.py index 59b36a012a239..e12613fbb8a66 100644 --- a/paddle/api/test/testGradientMachine.py +++ b/paddle/api/test/testGradientMachine.py @@ -30,8 +30,8 @@ def test_create_gradient_machine(self): self.assertIsNotNone(model_config) machine = swig_paddle.GradientMachine.createByModelConfig( model_config, swig_paddle.CREATE_MODE_NORMAL, - swig_paddle.ParameterOptimizer.create( - opt_config).getParameterTypes()) + swig_paddle.ParameterOptimizer.create(opt_config).getParameterTypes( + )) self.assertIsNotNone(machine) ipt, _ = util.loadMNISTTrainData() output = swig_paddle.Arguments.createArguments(0) @@ -43,7 +43,7 @@ def test_create_gradient_machine(self): assert isinstance(param, swig_paddle.Parameter) val = param.getBuf(swig_paddle.PARAMETER_VALUE) assert isinstance(val, swig_paddle.Vector) - arr = numpy.full((len(val),), 0.1, dtype="float32") + arr = numpy.full((len(val), ), 0.1, dtype="float32") val.copyFromNumpyArray(arr) param_config = param.getConfig().toProto() assert isinstance(param_config, diff --git a/paddle/api/test/testMatrix.py b/paddle/api/test/testMatrix.py index 2216ef30a58b0..2160612888b0f 100644 --- a/paddle/api/test/testMatrix.py +++ b/paddle/api/test/testMatrix.py @@ -69,7 +69,8 @@ def test_createDenseMat(self): def test_numpy(self): numpy_mat = np.matrix([[1, 2], [3, 4], [5, 6]], dtype="float32") m = swig_paddle.Matrix.createCpuDenseFromNumpy(numpy_mat) - self.assertEqual((int(m.getHeight()), int(m.getWidth())), numpy_mat.shape) + self.assertEqual((int(m.getHeight()), int(m.getWidth())), + numpy_mat.shape) # the numpy matrix and paddle matrix shared the same memory. numpy_mat[0, 1] = 342.23 diff --git a/paddle/api/test/testTrain.py b/paddle/api/test/testTrain.py index 7f79c2701e9ed..a3ba4eaaa69b3 100644 --- a/paddle/api/test/testTrain.py +++ b/paddle/api/test/testTrain.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from py_paddle import swig_paddle, DataProviderWrapperConverter +from py_paddle import swig_paddle import paddle.trainer.config_parser -from paddle.trainer.PyDataProviderWrapper import DenseSlot, IndexSlot import numpy import util @@ -99,7 +98,8 @@ def update_callback(param): cost_vec = outArgs.getSlotValue(0) assert isinstance(cost_vec, swig_paddle.Matrix) cost_vec = cost_vec.copyToNumpyMat() - print 'Finish Batch', batch_id, 'with cost ', cost_vec.sum() / batch_size + print 'Finish Batch', batch_id, 'with cost ', cost_vec.sum( + ) / batch_size batch_id += 1 for optimizer in optimizers: diff --git a/paddle/api/test/testTrainConfig.py b/paddle/api/test/testTrainConfig.py index 22148e31915da..77e0cd37d566d 100644 --- a/paddle/api/test/testTrainConfig.py +++ b/paddle/api/test/testTrainConfig.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=100, - learning_method=AdamOptimizer() -) +settings(batch_size=100, learning_method=AdamOptimizer()) din = data_layer(name='input', size=784) diff --git a/paddle/api/test/testTrainer.py b/paddle/api/test/testTrainer.py new file mode 100644 index 0000000000000..edd5a2da5785c --- /dev/null +++ b/paddle/api/test/testTrainer.py @@ -0,0 +1,63 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.config_parser import parse_config +from paddle.trainer.config_parser import logger +from py_paddle import swig_paddle +import util + + +def main(): + trainer_config = parse_config("./testTrainConfig.py", "") + model = swig_paddle.GradientMachine.createFromConfigProto( + trainer_config.model_config) + trainer = swig_paddle.Trainer.create(trainer_config, model) + trainer.startTrain() + for train_pass in xrange(2): + trainer.startTrainPass() + num = 0 + cost = 0 + while True: # Train one batch + batch_size = 1000 + data, atEnd = util.loadMNISTTrainData(batch_size) + if atEnd: + break + trainer.trainOneDataBatch(batch_size, data) + outs = trainer.getForwardOutput() + cost += sum(outs[0]['value']) + num += batch_size + trainer.finishTrainPass() + logger.info('train cost=%f' % (cost / num)) + + trainer.startTestPeriod() + num = 0 + cost = 0 + while True: # Test one batch + batch_size = 1000 + data, atEnd = util.loadMNISTTrainData(batch_size) + if atEnd: + break + trainer.testOneDataBatch(batch_size, data) + outs = trainer.getForwardOutput() + cost += sum(outs[0]['value']) + num += batch_size + trainer.finishTestPeriod() + logger.info('test cost=%f' % (cost / num)) + + trainer.finishTrain() + + +if __name__ == '__main__': + swig_paddle.initPaddle("--use_gpu=0", "--trainer_count=1") + main() diff --git a/paddle/api/test/testVector.py b/paddle/api/test/testVector.py index f5b5d0e32e420..5226df79eea3b 100644 --- a/paddle/api/test/testVector.py +++ b/paddle/api/test/testVector.py @@ -112,5 +112,6 @@ def testCopyFromNumpy(self): if __name__ == '__main__': - swig_paddle.initPaddle("--use_gpu=1" if swig_paddle.isGpuVersion() else "--use_gpu=0") + swig_paddle.initPaddle("--use_gpu=1" + if swig_paddle.isGpuVersion() else "--use_gpu=0") unittest.main() diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt old mode 100644 new mode 100755 index e03a9a1baa004..cdb730bb3cec7 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -2,10 +2,17 @@ set(AVX_SOURCES src/hl_math.cc src/hl_avx_functions.cc ) -set(CUDA_SOURCES - src/hl_time.cc - src/hl_cpu_functions.cc - ${AVX_SOURCES}) + +if(WITH_AVX) + set(CUDA_SOURCES + src/hl_time.cc + src/hl_cpu_functions.cc + ${AVX_SOURCES}) +else() + set(CUDA_SOURCES + src/hl_time.cc + src/hl_cpu_functions.cc) +endif() set(CUDA_CXX_WITH_GPU_SOURCES src/hl_cuda_cublas.cc diff --git a/paddle/cuda/include/hl_base.h b/paddle/cuda/include/hl_base.h index 77e2649b17214..9f80898a1f927 100644 --- a/paddle/cuda/include/hl_base.h +++ b/paddle/cuda/include/hl_base.h @@ -185,7 +185,7 @@ typedef struct { size_t nnz; } _hl_sparse_matrix_s, *hl_sparse_matrix_s; -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE /** * HPPL data type: real (float or double) * @@ -209,6 +209,15 @@ typedef struct { #define HL_FLOAT_MIN 2.2250738585072014e-308 #endif + +/** + * The maximum input value for exp, used to avoid overflow problem. + * + * Currently only used for tanh function. + */ +#define EXP_MAX_INPUT 40.0 + + /** * @brief DIVUP(x, y) is similar to ceil(x / y). * @note For CUDA, DIVUP will be used to specify @@ -245,4 +254,3 @@ extern __thread cudaStream_t default_stream; #endif /* __NVCC__ */ #endif /* HL_BASE_H_ */ - diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index dcae62d06b26d..70b5be6fda250 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -84,16 +84,25 @@ extern void hl_expand_feature2col( * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. * @param[out] tgtData output data. + * @param[in] tgtStride stride between output data samples. * */ extern void hl_maxpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData); + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride); /** * @brief Maximum pool backward. @@ -107,21 +116,29 @@ extern void hl_maxpool_forward( * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. - * @param[out] targetGrad output grad. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. * @param[in] scaleA scale. * @param[in] scaleB scale. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. + * @param[out] targetGrad output grad. + * @param[in] outStride stride between output data samples. * */ extern void hl_maxpool_backward( - int frameCnt, const real* inputData, + const int frameCnt, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB); + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* targetGrad, const int outStride); /** * @brief Averge pool forward. @@ -133,41 +150,58 @@ extern void hl_maxpool_backward( * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. * @param[out] tgtData output data. + * @param[in] tgtStride stride between output data samples. * */ extern void hl_avgpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData); + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride); /** * @brief Maximum pool backward. * * @param[in] frameCnt batch size of input image. - * @param[in] outGrad input data. + * @param[in] outGrad output grad data. * @param[in] channels number of channel. * @param[in] height image height. * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. - * @param[out] backGrad output grad. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. * @param[in] scaleA scale. * @param[in] scaleB scale. + * @param[out] backGrad output grad. + * @param[in] outStride stride between output data samples. * */ extern void hl_avgpool_backward( - int frameCnt, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* backGrad, - real scaleA, real scaleB); + const int frameCnt, const real* outGrad, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + int paddingH, int paddingW, + real scaleA, real scaleB, + real* backGrad, const int outStride); /** * @brief Cross-map-respose normalize forward. @@ -212,4 +246,98 @@ extern void hl_CMRNorm_backward( size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta); +/** + * @brief Bilinear interpolation forward. + * + * @param[in] inData input value. + * @param[in] inImgH input image height. + * @param[in] inImgW input image width. + * @param[in] inputH input batchSize. + * @param[in] inputW input image data dim. + * @param[out] outData output value. + * @param[in] outImgH output image height. + * @param[in] outImgW output image width. + * @param[in] outputH output batchSize. + * @param[in] outputW output image data dim. + * @param[in] numChannels number of channels. + * @param[in] ratioH inImgH / outImgH. + * @param[in] ratioW inImgW / outImgW. + * + */ +extern void hl_bilinear_forward(const real* inData, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* outData, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW); + + /** + * @brief Bilinear interpolation backward. + * + * @param[out] inGrad input gradient. + * @param[in] inImgH input image height. + * @param[in] inImgW input image width. + * @param[in] inputH input batchSize. + * @param[in] inputW input image data dim. + * @param[in] outGrad output gradient. + * @param[in] outImgH output image height. + * @param[in] outImgW output image width. + * @param[in] outputH output batchSize. + * @param[in] outputW output image data dim. + * @param[in] numChannels number of channels. + * @param[in] ratioH inImgH / outImgH. + * @param[in] ratioW inImgW / outImgW. + * + */ +extern void hl_bilinear_backward(real* inGrad, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* outGrad, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW); + +/** + * @brief MaxOut forward. + * + * @param[in] inData input data. + * @param[out] outData output data. + * @param[out] idData output maxId. + * @param[in] batchSize batchSize. + * @param[in] size number of channels * image height * image width. + * @param[in] featLen feature length = image height * image width. + * @param[in] groups number of groups. + */ +extern void hl_maxout_forward( + const real* inData, real* outData, int* idData, + size_t batchSize, size_t size, size_t featLen, size_t groups); + +/** + * @brief MaxOut backward. + * + * @param[out] inGrad input grad data. + * @param[in] outGrad output grad data. + * @param[in] idData output maxId. + * @param[in] batchSize batchSize. + * @param[in] size number of channels * image height * image width. + * @param[in] featLen feature length = image height * image width. + * @param[in] groups number of groups. + */ +extern void hl_maxout_backward( + real* inGrad, const real* outGrad, const int* idData, + size_t batchSize, size_t size, size_t featLen, size_t groups); + #endif /* HL_CNN_H_ */ diff --git a/paddle/cuda/include/hl_cpu_gru.cuh b/paddle/cuda/include/hl_cpu_gru.cuh index cba1c9f30da8d..d39cf67448b4f 100644 --- a/paddle/cuda/include/hl_cpu_gru.cuh +++ b/paddle/cuda/include/hl_cpu_gru.cuh @@ -20,7 +20,7 @@ limitations under the License. */ #include "paddle/math/MathFunctions.h" -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define CBLAS_GEMM paddle::gemm #else #define CBLAS_GEMM paddle::gemm diff --git a/paddle/cuda/include/hl_cuda_cublas.h b/paddle/cuda/include/hl_cuda_cublas.h index 0ffbed18b5f9e..d757317eb4a97 100644 --- a/paddle/cuda/include/hl_cuda_cublas.h +++ b/paddle/cuda/include/hl_cuda_cublas.h @@ -21,8 +21,8 @@ limitations under the License. */ /** * @brief Matrix transpose: C_d = T(A_d) * - * @param[in] A_d input matrix (M x N). - * @param[out] C_d output matrix (N x M). + * @param[in] A_d input matrix (dimM x dimN). + * @param[out] C_d output matrix (dimN x dimM). * @param[in] dimM matrix height. * @param[in] dimN matrix width. * @param[in] lda the first dimension of A_d. @@ -39,8 +39,8 @@ extern void hl_matrix_transpose(real *A_d, /* * @brief Matrix transpose, while lda = dimN, ldc = dimM. * - * @param[in] A_d input matrix (M x N). - * @param[out] C_d output matrix (N x M). + * @param[in] A_d input matrix (dimM x dimN). + * @param[out] C_d output matrix (dimN x dimM). * @param[in] dimM matrix height. * @param[in] dimN matrix width. * @@ -50,6 +50,22 @@ extern void hl_matrix_transpose(real *A_d, int dimM, int dimN); +/* + * @brief Matrix inverse + * + * @param[in] A_d input matrix (dimN x dimN). + * @param[out] C_d output matrix (dimN x dimN). + * @param[in] dimN matrix height = matrix width + * @param[in] lda the first dimension of A_d + * @param[in] ldc the first dimension of C_d + * + */ +extern void hl_matrix_inverse(real *A_d, + real *C_d, + int dimN, + int lda, + int ldc); + /** * @brief C_d = alpha*(op(A_d) * op(B_d)) + beta*C_d * diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh index 88d950d6c1713..159c26f443cb1 100755 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -48,5 +48,24 @@ inline __device__ double paddleAtomicAdd(double* address, double val) { } } // namespace paddle +/** + * @brief sum reduction + * + * @param[in,out] smem input data, better to use __shared__ memory. + * @param[in] tid thread index. + * @param[in] threads the total thread number used to reduce, + * such as, blockDim.x. + * + * @return smem[0]: the sum of each elements in smem. + */ +__device__ __forceinline__ +void simpleReduce(real* smem, int tid, int threads) { + for (unsigned int s = threads / 2; s > 0; s >>= 1) { + if (tid < s) { + smem[tid] += smem[tid + s]; + } + __syncthreads(); + } +} #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ diff --git a/paddle/cuda/include/hl_gpu_functions.cuh b/paddle/cuda/include/hl_gpu_functions.cuh index 38df4eb8958f2..a2c5ebd18a440 100644 --- a/paddle/cuda/include/hl_gpu_functions.cuh +++ b/paddle/cuda/include/hl_gpu_functions.cuh @@ -28,7 +28,7 @@ namespace hppl { const real min = SIGMOID_THRESHOLD_MIN; const real max = SIGMOID_THRESHOLD_MAX; real tmp = (a < min) ? min : ((a > max) ? max : a); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE return __fdividef(1.0f, 1.0f + __expf(-tmp)); #else return 1.0 / (1.0 + exp(-tmp)); @@ -36,7 +36,7 @@ namespace hppl { } __device__ static real tanh(const real a) { -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE return __fdividef(2.0f, (1.0f + __expf(-2.0f*a))) - 1.0f; #else return (2.0 / (1.0 + exp(-2.0*a))) - 1.0; diff --git a/paddle/cuda/include/hl_matrix.h b/paddle/cuda/include/hl_matrix.h index 17419790471a7..6195e30b9974d 100644 --- a/paddle/cuda/include/hl_matrix.h +++ b/paddle/cuda/include/hl_matrix.h @@ -126,6 +126,36 @@ extern void hl_matrix_cross_entropy_bp(real* grad_d, int dimM, int dimN); +/** + * @brief Matrix multi-binary label cross entropy + * + * @param[in] output input matrix (M x N). + * @param[out] entropy output matrix (M x 1). + * @param[in] mat input sparse matrix. + * @param[in] dimM matrix height. + * @param[in] dimN matrix width. + */ +extern void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s mat, + int dimM, + int dimN); + +/** + * @brief Matrix multi-binary label cross entropy backprop + * + * @param[in] output input matrix (M x N). + * @param[out] grad output matrix (M x N). + * @param[in] mat input sparse matrix. + * @param[in] dimM matrix height. + * @param[in] dimN matrix width. + */ +extern void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s mat, + int dimM, + int dimN); + /** * @brief Matrix zero memory. * @@ -229,4 +259,40 @@ extern void hl_cossim_derivative(real* grad, int input2_height, real scale); +/** + * @brief Matrix addition: A_d[i][j] += scale * B_d[j/channel]. + * + * @param[in] A_d input matrix (M x N). + * @param[in] B_d input matrix (1 x channel). + * @param[in] channel width of B. + * @param[in] dimM height of A. + * @param[in] dimN width of A. + * @param[in] scale scalar used for addition. + * + */ +extern void hl_matrix_add_shared_bias(real* A_d, + real* B_d, + const int channel, + const int dimM, + const int dimN, + real scale); + +/** + * @brief Matrix addition: A_d[i][j] += scale * B_d[j/channel]. + * + * @param[in] B_d input matrix (1 x channel). + * @param[in] A_d input matrix (M x N). + * @param[in] channel width of B. + * @param[in] dimM height of A. + * @param[in] dimN width of A. + * @param[in] scale scalar used for addition. + * + */ +extern void hl_matrix_collect_shared_bias(real* B_d, + real* A_d, + const int channel, + const int dimM, + const int dimN, + real scale); + #endif /* HL_MATRIX_H_ */ diff --git a/paddle/cuda/include/hl_matrix_base.cuh b/paddle/cuda/include/hl_matrix_base.cuh index 473d394c0c688..a3645ef51e6ef 100644 --- a/paddle/cuda/include/hl_matrix_base.cuh +++ b/paddle/cuda/include/hl_matrix_base.cuh @@ -30,7 +30,7 @@ limitations under the License. */ #define INLINE inline #endif -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define DEVICE_FMAX fmaxf #define DEVICE_FMIN fminf #else diff --git a/paddle/cuda/include/hl_matrix_type.cuh b/paddle/cuda/include/hl_matrix_type.cuh index 6917f36290141..51e483d1fb2ff 100644 --- a/paddle/cuda/include/hl_matrix_type.cuh +++ b/paddle/cuda/include/hl_matrix_type.cuh @@ -21,7 +21,7 @@ limitations under the License. */ #ifdef __CUDA_ARCH__ // typedef void* vecType; #include -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE typedef float4 vecType; #else typedef double2 vecType; @@ -30,7 +30,7 @@ typedef double2 vecType; #include #include #include -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE typedef __m128 vecType; #else typedef __m128d vecType; diff --git a/paddle/cuda/include/hl_sequence.h b/paddle/cuda/include/hl_sequence.h index 828c21beb2fbd..46d86b2982f06 100644 --- a/paddle/cuda/include/hl_sequence.h +++ b/paddle/cuda/include/hl_sequence.h @@ -143,7 +143,7 @@ extern void hl_context_projection_backward_weight(real* outputGrad, */ extern void hl_sequence2batch_copy(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount, bool seq2batch); diff --git a/paddle/cuda/include/hl_sparse.h b/paddle/cuda/include/hl_sparse.h index 22f7a228e0ad6..9acdebdebf377 100644 --- a/paddle/cuda/include/hl_sparse.h +++ b/paddle/cuda/include/hl_sparse.h @@ -223,6 +223,7 @@ extern void hl_matrix_csc2dense(hl_sparse_matrix_s A_d, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * @note transb is not support HPPL_OP_T. * @@ -251,6 +252,7 @@ extern void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * @note transb is not support HPPL_OP_T. * @@ -275,6 +277,7 @@ extern void hl_matrix_csc_mul_dense(hl_sparse_matrix_s A_d, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * @note transa is not support HPPL_OP_T. * @@ -327,6 +330,7 @@ extern void hl_sparse_matrix_mul(real* A_d, hl_trans_op_t transa, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * * @note transa is not support HPPL_OP_T. diff --git a/paddle/cuda/include/hl_sse_matrix_kernel.cuh b/paddle/cuda/include/hl_sse_matrix_kernel.cuh index c90d49e4adeb5..45db2f313e0d6 100644 --- a/paddle/cuda/include/hl_sse_matrix_kernel.cuh +++ b/paddle/cuda/include/hl_sse_matrix_kernel.cuh @@ -20,7 +20,7 @@ limitations under the License. */ #define VECTOR_SIZE 16 -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE /* number of float in vector */ #define VECTOR_LEN 4 #define VECTOR_SET _mm_set_ps1 @@ -41,7 +41,7 @@ inline bool hl_check_align(void *ptr) { return hl_check_align(reinterpret_cast(ptr)); } -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE template inline real hl_agg_op(Agg agg, vecType mm) { __m128 lo = _mm_unpacklo_ps(mm, mm); diff --git a/paddle/cuda/include/stub/hl_cnn_stub.h b/paddle/cuda/include/stub/hl_cnn_stub.h index e4d46e4fb186e..c6f32ad337705 100644 --- a/paddle/cuda/include/stub/hl_cnn_stub.h +++ b/paddle/cuda/include/stub/hl_cnn_stub.h @@ -38,29 +38,47 @@ inline void hl_expand_feature2col( real* dataCol) {} inline void hl_maxpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) {} + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) {} inline void hl_maxpool_backward( - int frameCnt, const real* inputData, + const int frameCnt, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB) {} + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* targetGrad, const int outStride) {} inline void hl_avgpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) {} + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) {} inline void hl_avgpool_backward( - int frameCnt, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* backGrad, - real scaleA, real scaleB) {} + const int frameCnt, const real* outGrad, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + int paddingH, int paddingW, + real scaleA, real scaleB, + real* backGrad, const int outStride) {} inline void hl_CMRNorm_forward( size_t frameCnt, const real* in, real* scale, real* out, @@ -73,4 +91,40 @@ inline void hl_CMRNorm_backward( size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta) {} +inline void hl_bilinear_forward(const real* inData, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* outData, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) {} + +inline void hl_bilinear_backward(real* inGrad, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* outGrad, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) {} + +inline void hl_maxout_forward( + const real* inData, real* outData, int* idData, + size_t batchSize, size_t size, size_t featLen, size_t group) {} + +inline void hl_maxout_backward( + real* inGrad, const real* outGrad, const int* idData, + size_t batchSize, size_t size, size_t featLen, size_t group) {} + #endif // HL_CNN_STUB_H_ diff --git a/paddle/cuda/include/stub/hl_cuda_cublas_stub.h b/paddle/cuda/include/stub/hl_cuda_cublas_stub.h index 4a5e2a25a71b3..903dcbe8355d6 100644 --- a/paddle/cuda/include/stub/hl_cuda_cublas_stub.h +++ b/paddle/cuda/include/stub/hl_cuda_cublas_stub.h @@ -30,6 +30,12 @@ inline void hl_matrix_transpose(real *A_d, int dimM, int dimN) {} +inline void hl_matrix_inverse(real *A_d, + real *C_d, + int dimN, + int lda, + int ldc) {} + inline void hl_matrix_mul(real *A_d, hl_trans_op_t transa, real *B_d, hl_trans_op_t transb, real *C_d, diff --git a/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h b/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h index 34c173908246e..b96804afd86ba 100644 --- a/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h +++ b/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h @@ -199,4 +199,3 @@ inline void hl_batch_norm_backward(hl_tensor_descriptor inputDesc, real *savedInvVar) {} #endif // HL_CUDA_CUDNN_STUB_H_ - diff --git a/paddle/cuda/include/stub/hl_matrix_stub.h b/paddle/cuda/include/stub/hl_matrix_stub.h index f1f1020c84d46..76cac2e577693 100644 --- a/paddle/cuda/include/stub/hl_matrix_stub.h +++ b/paddle/cuda/include/stub/hl_matrix_stub.h @@ -57,6 +57,18 @@ inline void hl_matrix_cross_entropy_bp(real* grad_d, int dimM, int dimN) {} +inline void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s mat, + int dimM, + int dimN) {} + +inline void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s mat, + int dimM, + int dimN) {} + inline void hl_matrix_zero_mem(real* data, int num) {} inline void hl_param_relu_forward(real* output, @@ -101,4 +113,17 @@ inline void hl_cossim_derivative(real* grad, int input2_height, real scale) {} +inline void hl_matrix_add_shared_bias(real* A_d, + real* B_d, + const int channel, + const int dimM, + const int dimN, + real scale) {} + +inline void hl_matrix_collect_shared_bias(real* B_d, + real* A_d, + const int channel, + const int dimM, + const int dimN, + real scale) {} #endif // HL_MATRIX_STUB_H_ diff --git a/paddle/cuda/include/stub/hl_sequence_stub.h b/paddle/cuda/include/stub/hl_sequence_stub.h index 417f40e0a69f6..aabd956c37f7d 100644 --- a/paddle/cuda/include/stub/hl_sequence_stub.h +++ b/paddle/cuda/include/stub/hl_sequence_stub.h @@ -62,7 +62,7 @@ inline void hl_context_projection_backward_weight(real* outputGrad, inline void hl_sequence2batch_copy(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount, bool seq2batch) {} diff --git a/paddle/cuda/src/avx_mathfun.h b/paddle/cuda/src/avx_mathfun.h index 808c2508d1a1a..2922d4dc29376 100644 --- a/paddle/cuda/src/avx_mathfun.h +++ b/paddle/cuda/src/avx_mathfun.h @@ -718,4 +718,3 @@ void sincos256_ps(v8sf x, v8sf *s, v8sf *c) { *s = _mm256_xor_ps(xmm1, sign_bit_sin); *c = _mm256_xor_ps(xmm2, sign_bit_cos); } - diff --git a/paddle/cuda/src/hl_avx_functions.cc b/paddle/cuda/src/hl_avx_functions.cc index 2d471206f61f2..08976180fff5b 100644 --- a/paddle/cuda/src/hl_avx_functions.cc +++ b/paddle/cuda/src/hl_avx_functions.cc @@ -38,7 +38,9 @@ namespace hppl { } __m256 tanh(const __m256 a) { + __m256 max = _mm256_set1_ps(EXP_MAX_INPUT); __m256 tmp = _mm256_mul_ps(_mm256_set1_ps(-2.0f), a); + tmp = _mm256_min_ps(tmp, max); tmp = exp(tmp); return _mm256_sub_ps( _mm256_div_ps(_mm256_set1_ps(2.0f), diff --git a/paddle/cuda/src/hl_cpu_functions.cc b/paddle/cuda/src/hl_cpu_functions.cc index 3fd6b278d0537..b8352c2d537fb 100644 --- a/paddle/cuda/src/hl_cpu_functions.cc +++ b/paddle/cuda/src/hl_cpu_functions.cc @@ -30,7 +30,9 @@ namespace hppl { } real tanh(const real a) { - return (2.0 / (1.0 + exp(-2.0*a))) - 1.0; + real tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + return (2.0 / (1.0 + exp(tmp))) - 1.0; } real linear(const real a) { diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index b3695a2c7f88e..ae387a8bc0e07 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -145,72 +145,88 @@ void hl_shrink_col2feature(const real * dataCol, size_t channels, CHECK_SYNC("hl_shrink_col2feature failed"); } -__global__ void KeMaxPoolForward(int nthreads, const real* inputData, - int channels, int height, int width, - int pooledH, int pooledW, - int ksize, int stride, int start, - real* tgtData) { - int index = blockIdx.y * blockDim.x + threadIdx.x; +__global__ void KeMaxPoolForward(const int nthreads, const real* inputData, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int ksizeW, const int ksizeH, + const int strideH, const int strideW, + const int offsetH, const int offsetW, + real* tgtData, const int tgtStride) { + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; int ph = (index / pooledW) % pooledH; int c = (index / pooledW / pooledH) % channels; - int frameNum = blockIdx.x; - int hstart = ph * stride + start; - int hend = min(hstart + ksize, height); - int wstart = pw * stride + start; - int wend = min(wstart + ksize, width); + int frameNum = index / pooledW / pooledH / channels; + int hstart = ph * strideH - offsetH; + int wstart = pw * strideW - offsetW; + int hend = min(hstart + ksizeH, height); + int wend = min(wstart + ksizeW, width); + hstart = max(hstart, 0); + wstart = max(wstart, 0); real maxval = -FLT_MAX; inputData += (frameNum * channels + c) * height * width; - tgtData += (frameNum * channels) * pooledW * pooledH; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { if (maxval < inputData[h * width + w]) maxval = inputData[h * width + w]; } } - tgtData[index] = maxval; + int tgtIndex = index % (pooledW * pooledH * channels) + + frameNum * tgtStride; + tgtData[tgtIndex] = maxval; } } -void hl_maxpool_forward(int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) { - int num_kernels = pooledH * pooledW * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; +void hl_maxpool_forward(const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) { + + int num_kernels = pooledH * pooledW * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); + dim3 grid(blocks, 1); + KeMaxPoolForward<<< grid, threads, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, - pooledH, pooledW, sizeX, stride, start, tgtData); + pooledH, pooledW, sizeX, sizeY, strideH, strideW, + paddingH, paddingW, tgtData, tgtStride); CHECK_SYNC("hl_maxpool_forward failed"); } -__global__ void KeMaxPoolBackward(int nthreads, const real* inputData, +__global__ void KeMaxPoolBackward(const int nthreads, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB) { - int index = blockIdx.y * blockDim.x + threadIdx.x; + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int padH, const int padW, + real scaleA, real scaleB, + real* targetGrad, const int outStride) { + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { // find out the local index // find out the local offset - int offsetW = index % width + start; - int offsetH = (index / width) % height + start; + int offsetW = index % width + padW; + int offsetH = (index / width) % height + padH; int offsetC = (index / width / height) % channels; - int frameNum = blockIdx.x; - int phstart = (offsetH < sizeX) ? 0 : (offsetH - sizeX) / stride + 1; - int phend = min(offsetH / stride + 1, pooledH); - int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / stride + 1; - int pwend = min(offsetW / stride + 1, pooledW); + + int frameNum = index / width / height / channels; + int phstart = (offsetH < sizeY) ? 0 : (offsetH - sizeY) / strideH + 1; + int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / strideW + 1; + int phend = offsetH >= 0 ? min(offsetH / strideH + 1, pooledH) : 0; + int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; - inputData += (frameNum * channels) * height * width; real input = inputData[index]; - outData += (frameNum * channels + offsetC) * pooledH * pooledW; - outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; - targetGrad += (frameNum * channels) * height * width; + outData += (frameNum * outStride + offsetC * pooledH * pooledW); + outGrad += (frameNum * outStride + offsetC * pooledH * pooledW); for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { if (input == outData[ph * pooledW + pw]) { @@ -223,90 +239,118 @@ __global__ void KeMaxPoolBackward(int nthreads, const real* inputData, } } -void hl_maxpool_backward(int frameCnt, const real* inputData, +void hl_maxpool_backward(const int frameCnt, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB) { - int num_kernels = (height - start) * (width - start) * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; - dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* targetGrad, const int outStride) { - KeMaxPoolBackward<<< grid, threads, 0, STREAM_DEFAULT >>> + int num_kernels = height * width * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; + + KeMaxPoolBackward<<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, inputData, outData, outGrad, channels, - height, width, pooledH, pooledW, sizeX, stride, start, - targetGrad, scaleA, scaleB); + height, width, pooledH, pooledW, sizeX, sizeY, + strideH, strideW, + paddingH, paddingW, + scaleA, scaleB, + targetGrad, outStride); CHECK_SYNC("hl_maxpool_backward"); } -__global__ void KeAvePoolForward(int nthreads, const real* inputData, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* tgtData) { - int index = blockIdx.y * blockDim.x + threadIdx.x; +__global__ void KeAvgPoolForward(const int nthreads, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int padH, const int padW, + real* tgtData, const int tgtStride) { + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; int ph = (index / pooledW) % pooledH; int c = (index / pooledW / pooledH) % channels; - int frameNum = blockIdx.x; - int hstart = ph * stride + start; - int hend = min(hstart + sizeX, height); - int wstart = pw * stride + start; - int wend = min(wstart + sizeX, width); + int frameNum = index / pooledW / pooledH / channels; + + int hstart = ph * strideH - padH; + int wstart = pw * strideW - padW; + int hend = min(hstart + sizeY, height + padH); + int wend = min(wstart + sizeX, width + padW); + int pool_size = (hend - hstart) * (wend - wstart); + hstart = max(hstart, 0); + wstart = max(wstart, 0); + hend = min(hend, height); + wend = min(wend, width); + real aveval = 0; inputData += (frameNum * channels + c) * height * width; - tgtData += (frameNum * channels) * pooledH * pooledW; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { aveval += inputData[h * width + w]; } } - tgtData[index] = aveval / ((hend - hstart) * (wend - wstart)); + int tgtIndex = index % (pooledW * pooledH * channels) + + frameNum * tgtStride; + tgtData[tgtIndex] = aveval / pool_size; } } -void hl_avgpool_forward(int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) { - int num_kernels = pooledH * pooledW * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; - dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); - KeAvePoolForward<<< grid, threads, 0, STREAM_DEFAULT >>> +void hl_avgpool_forward(const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) { + int num_kernels = pooledH * pooledW * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; + KeAvgPoolForward<<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, pooledH, pooledW, - sizeX, stride, start, tgtData); + sizeX, sizeY, strideH, strideW, + paddingH, paddingW, tgtData, tgtStride); CHECK_SYNC("hl_avgpool_forward failed"); } -__global__ void KeAvgPoolBackward(int nthreads, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* tgtGrad, - real scaleA, real scaleB) { - int index = blockIdx.y * blockDim.x + threadIdx.x; +__global__ void KeAvgPoolBackward(const int nthreads, const real* outGrad, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int padH, const int padW, + real scaleA, real scaleB, + real* tgtGrad, const int outStride) { + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { - int offsetW = index % width + start; - int offsetH = (index / width) % height + start; + int offsetW = index % width + padW; + int offsetH = (index / width) % height + padH; int offsetC = (index / width / height) % channels; - int frameNum = blockIdx.x; - int phstart = (offsetH < sizeX) ? 0 : (offsetH - sizeX) / stride + 1; - int phend = min(offsetH / stride + 1, pooledH); - int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / stride + 1; - int pwend = min(offsetW / stride + 1, pooledW); + int frameNum = index / width / height / channels; + + int phstart = (offsetH < sizeY) ? 0 : (offsetH - sizeY) / strideH + 1; + int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / strideW + 1; + int phend = offsetH >= 0 ? min(offsetH / strideH + 1, pooledH) : 0; + int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; - outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; - tgtGrad += (frameNum * channels) * height * width; + outGrad += (frameNum * outStride + offsetC * pooledH * pooledW); + for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { // figure out the pooling size - int poolsize = (min(ph * stride + sizeX, height) - ph * stride) * - (min(pw * stride + sizeX, width) - pw * stride); + int hstart = ph * strideH - padH; + int wstart = pw * strideW - padW; + int hend = min(hstart + sizeY, height + padH); + int wend = min(wstart + sizeX, width + padW); + int poolsize = (hend - hstart) * (wend - wstart); gradient += outGrad[ph * pooledW + pw]/poolsize; } } @@ -314,20 +358,25 @@ __global__ void KeAvgPoolBackward(int nthreads, const real* outGrad, } } -void hl_avgpool_backward(int frameCnt, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* backGrad, - real scaleA, real scaleB) { - int num_kernels = (height - start) * (width - start) * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; - dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); +void hl_avgpool_backward(const int frameCnt, const real* outGrad, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* backGrad, const int outStride) { + int num_kernels = height * width * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; - KeAvgPoolBackward <<< grid, threads, 0, STREAM_DEFAULT >>> + KeAvgPoolBackward <<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, outGrad, channels, height, width, - pooledH, pooledW, sizeX, stride, start, backGrad, scaleA, scaleB); + pooledH, pooledW, sizeX, sizeY, + strideH, strideW, + paddingH, paddingW, + scaleA, scaleB, + backGrad, outStride); CHECK_SYNC("hl_avgpool_backward failed"); } @@ -479,7 +528,7 @@ void hl_CMRNorm_backward(size_t frameCnt, const real* inV, size_t height, size_t width, size_t sizeX, real alpha, real beta) { size_t threadsNum = frameCnt * height * width; - size_t blocksX = (threadsNum + 1024 -1) / 1024; + size_t blocksX = (threadsNum + 1024 - 1) / 1024; size_t blocksY = 1; dim3 threads(1024, 1); dim3 grid(blocksX, blocksY); @@ -488,3 +537,194 @@ void hl_CMRNorm_backward(size_t frameCnt, const real* inV, height, width, sizeX, alpha, beta, inDiff); CHECK_SYNC("hl_CMRNorm_backward"); } + +__global__ void KeBilinearInterpFw(const real* in, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* out, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + int nthreads = outputH * outputW; + int tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid < nthreads) { + int outIdH = tid / outputW; + int outIdW = tid % outputW; + int inImgSize = inputW / numChannels; + int outImgSize = outputW / numChannels; + int channelId = outIdW / outImgSize; + + int outImgIdy = (outIdW % outImgSize) / outImgW; + int inImgIdy = ratioH * outImgIdy; + int hId = (inImgIdy < inImgH - 1) ? 1 : 0; + real h1lambda = ratioH * outImgIdy - inImgIdy; + real h2lambda = 1.f - h1lambda; + + int outImgIdx = tid % outImgW; + int inImgIdx = ratioW * outImgIdx; + int wId = (inImgIdx < inImgW - 1) ? 1 : 0; + real w1lambda = ratioW * outImgIdx - inImgIdx; + real w2lambda = 1.f - w1lambda; + + const real* inPos = + &in[outIdH * inputW + channelId * inImgSize + inImgIdy * inImgW + inImgIdx]; + + // bilinear interpolation + out[outIdH * outputW + outIdW] = + h2lambda * (w2lambda * inPos[0] + w1lambda * inPos[wId]) + + h1lambda * (w2lambda * inPos[hId * inImgW] + w1lambda * inPos[hId * inImgW + wId]); + } +} + +void hl_bilinear_forward(const real* inData, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* outData, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + int threadNum = outputH * outputW; + int blocks = (threadNum + 1024 - 1) / 1024; + + KeBilinearInterpFw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( + inData, inImgH, inImgW, inputH, inputW, outData, outImgH, + outImgW, outputH, outputW, numChannels, ratioH, ratioW); + CHECK_SYNC("hl_bilinear_forward failed"); +} + +__global__ void KeBilinearInterpBw(real* in, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* out, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + int nthreads = outputH * outputW; + int tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid < nthreads) { + int outIdH = tid / outputW; + int outIdW = tid % outputW; + int inImgSize = inputW / numChannels; + int outImgSize = outputW / numChannels; + int channelId = outIdW / outImgSize; + + int outImgIdy = (outIdW % outImgSize) / outImgW; + int inImgIdy = ratioH * outImgIdy; + int hId = (inImgIdy < inImgH - 1) ? 1 : 0; + real h1lambda = ratioH * outImgIdy - inImgIdy; + real h2lambda = 1.f - h1lambda; + + int outImgIdx = tid % outImgW; + int inImgIdx = ratioW * outImgIdx; + int wId = (inImgIdx < inImgW - 1) ? 1 : 0; + real w1lambda = ratioW * outImgIdx - inImgIdx; + real w2lambda = 1.f - w1lambda; + + real* inPos = + &in[outIdH * inputW + channelId * inImgSize + inImgIdy * inImgW + inImgIdx]; + const real* outPos = &out[outIdH * outputW + outIdW]; + atomicAdd(&inPos[0], h2lambda * w2lambda * outPos[0]); + atomicAdd(&inPos[wId], h2lambda * w1lambda * outPos[0]); + atomicAdd(&inPos[hId * inImgW], h1lambda * w2lambda * outPos[0]); + atomicAdd(&inPos[hId * inImgW + wId], h1lambda * w1lambda * outPos[0]); + } +} + +void hl_bilinear_backward(real* inGrad, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* outGrad, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + int threadNum = outputH * outputW; + int blocks = (threadNum + 1024 - 1) / 1024; + + KeBilinearInterpBw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( + inGrad, inImgH, inImgW, inputH, inputW, outGrad, outImgH, + outImgW, outputH, outputW, numChannels, ratioH, ratioW); + CHECK_SYNC("hl_bilinear_backward failed"); +} + +__global__ void maxoutFpCompute(size_t nthreads, const real * inData, + real * outData, int* idData, + size_t size, size_t featLen, size_t groups) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if(index < nthreads) { + size_t batch_idx = index / size; + size_t i = index % size; + size_t channel_idx = i / featLen; + size_t feat_idx = i % featLen; + size_t data_idx = (batch_idx * size + channel_idx * featLen) * groups + feat_idx; + real max = inData[data_idx]; + int maxId = 0; + for (size_t g = 1; g < groups; ++g) { + real tmp = inData[data_idx + g * featLen]; + if (tmp > max) { + max = tmp; + maxId = g; + } + } + outData[index] = max; + idData[index] = maxId; + } +} + +void hl_maxout_forward(const real* inData, real* outData, + int* idData, size_t batchSize, size_t size, + size_t featLen, size_t groups) { + int num_kernels = size * batchSize; + int blocks = (num_kernels + 1024 - 1) / 1024; + maxoutFpCompute<<< blocks, 1024, 0, STREAM_DEFAULT>>>( + num_kernels, inData, outData, idData, size, featLen, groups); + CHECK_SYNC("hl_maxout_forward failed"); +} + +__global__ void maxoutBpCompute(size_t nthreads, real* inGrad, + const real* outGrad, const int* idData, + size_t size, size_t featLen, size_t groups) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if(index < nthreads) { + size_t batch_idx = index / size; + size_t i = index % size; + size_t channel_idx = i / featLen; + size_t feat_idx = i % featLen; + size_t newIndex = batch_idx * size; + size_t gradIdx = (channel_idx * groups + (idData + newIndex)[i]) * featLen + feat_idx; + (inGrad + newIndex * groups)[gradIdx] += (outGrad + newIndex)[i]; + } +} + +void hl_maxout_backward(real* inGrad, const real* outGrad, + const int* idData, size_t batchSize, size_t size, + size_t featLen, size_t groups) { + int num_kernels = size * batchSize; + int blocks = (num_kernels + 1024 - 1) / 1024; + maxoutBpCompute<<< blocks, 1024, 0, STREAM_DEFAULT >>>( + num_kernels, inGrad, outGrad, idData, size, featLen, groups); + CHECK_SYNC("hl_maxout_backward failed"); +} diff --git a/paddle/cuda/src/hl_cuda_cublas.cc b/paddle/cuda/src/hl_cuda_cublas.cc index dc109487ded20..f16376ec937d3 100644 --- a/paddle/cuda/src/hl_cuda_cublas.cc +++ b/paddle/cuda/src/hl_cuda_cublas.cc @@ -15,6 +15,7 @@ limitations under the License. */ #include #include +#include "hl_cuda.h" #include "hl_cuda_cublas.h" #include "hl_thread.ph" #include "hl_dso_loader.h" @@ -75,6 +76,10 @@ DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasCgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasZgemmBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetrfBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetriBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetrfBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetriBatched) CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) #undef DYNAMIC_LOAD_CUBLAS_WRAP @@ -84,14 +89,18 @@ CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) } /* namespace dynload */ -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define CUBLAS_GEAM dynload::cublasSgeam #define CUBLAS_GEMV dynload::cublasSgemv #define CUBLAS_GEMM dynload::cublasSgemm +#define CUBLAS_GETRF dynload::cublasSgetrfBatched +#define CUBLAS_GETRI dynload::cublasSgetriBatched #else #define CUBLAS_GEAM dynload::cublasDgeam #define CUBLAS_GEMV dynload::cublasDgemv #define CUBLAS_GEMM dynload::cublasDgemm +#define CUBLAS_GETRF dynload::cublasDgetrfBatched +#define CUBLAS_GETRI dynload::cublasDgetriBatched #endif const char* hl_cublas_get_error_string(cublasStatus_t status) { @@ -162,6 +171,54 @@ void hl_matrix_transpose(real *A_d, real *C_d, int dimM, int dimN) { hl_matrix_transpose(A_d, C_d, dimM, dimN, dimN, dimM); } +void hl_matrix_inverse(real *A_d, real *C_d, int dimN, int lda, int ldc) { + /* Solve Ax = I */ + CHECK_NOTNULL(A_d); + CHECK_NOTNULL(C_d); + + /* Step 1: Compute the LU decomposition of matrix A */ + real **inout_h = &A_d; + real **inout_d = (real **)hl_malloc_device(sizeof(real *)); + hl_memcpy(inout_d, inout_h, sizeof(real *)); + + int *pivot_d = (int *)hl_malloc_device(dimN*sizeof(int)); + int *info_d = (int *)t_resource.gpu_mem; + + /* Note: cublasSgetrfBatched is used to calculate a number of + small-sized matrices. There may be a better way to reconstruct + the API for better performance. + */ + CHECK_CUBLAS(CUBLAS_GETRF(t_resource.handle, + dimN, inout_d, lda, pivot_d, + info_d, 1)); + + int info_h; + hl_memcpy(&info_h, info_d, sizeof(int)); + if (info_h != 0) { + LOG(FATAL) << "Factorization of matrix failed: matrix may be singular.\n"; + } + + /* Step 2: Compute the inverse of the matrix given its LU decomposition */ + real **out_h = &C_d; + real **out_d = (real **)hl_malloc_device(sizeof(real *)); + hl_memcpy(out_d, out_h, sizeof(real *)); + + CHECK_CUBLAS(CUBLAS_GETRI(t_resource.handle, + dimN, (const real **)inout_d, lda, pivot_d, + out_d, ldc, info_d, 1)); + + hl_memcpy(&info_h, info_d, sizeof(int)); + if (info_h != 0) { + LOG(FATAL) << "Inversion of matrix failed: matrix may be singular.\n"; + } + + hl_free_mem_device(inout_d); + hl_free_mem_device(pivot_d); + hl_free_mem_device(out_d); + + CHECK_SYNC("hl_matrix_inverse failed"); +} + void hl_matrix_mul(real *A_d, hl_trans_op_t transa, real *B_d, hl_trans_op_t transb, real *C_d, diff --git a/paddle/cuda/src/hl_cuda_cudnn.cc b/paddle/cuda/src/hl_cuda_cudnn.cc index c2dce1977bdf5..92b28e4345c3d 100644 --- a/paddle/cuda/src/hl_cuda_cudnn.cc +++ b/paddle/cuda/src/hl_cuda_cudnn.cc @@ -20,6 +20,11 @@ limitations under the License. */ #include "hl_thread.ph" #include "hl_dso_loader.h" #include "paddle/utils/Logging.h" +#include "paddle/utils/CommandLineParser.h" + +P_DEFINE_int32(cudnn_conv_workspace_limit_in_mb, 4096, + "Specify cuDNN max workspace limit, in units MB, " + "4096MB=4GB by default."); namespace dynload { @@ -36,65 +41,28 @@ void* cudnn_dso_handle = nullptr; #ifdef PADDLE_USE_DSO -#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - cudnnStatus_t operator()(Args... args) { \ - typedef cudnnStatus_t (*cudnnFunc)(Args...); \ - std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, \ - &cudnn_dso_handle); \ - void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ +#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using cudnn_func = decltype(__name(args...))(*)(Args...); \ + std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, \ + &cudnn_dso_handle); \ + void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ } __name; /* struct DynLoad__##__name */ -struct DynLoad__cudnnGetVersion { - template - size_t operator()(Args... args) { - typedef size_t (*cudnnFunc)(Args...); - std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, - &cudnn_dso_handle); - void* p_name = dlsym(cudnn_dso_handle, "cudnnGetVersion"); - return reinterpret_cast(p_name)(args...); - } -} cudnnGetVersion; /* struct DynLoad__##__name */ - -struct DynLoad__cudnnGetErrorString { - template - const char* operator()(Args... args) { - typedef const char* (*cudnnFunc)(Args...); - std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, - &cudnn_dso_handle); - void* p_name = dlsym(cudnn_dso_handle, "cudnnGetErrorString"); - return reinterpret_cast(p_name)(args...); - } -} cudnnGetErrorString; /* struct DynLoad__##__name */ - - #else -#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - cudnnStatus_t operator()(Args... args) { \ - return __name(args...); \ - } \ +#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + return __name(args...); \ + } \ } __name; /* struct DynLoad__##__name */ -struct DynLoad__cudnnGetVersion { - template - size_t operator()(Args... args) { - return cudnnGetVersion(args...); - } -} cudnnGetVersion; /* struct DynLoad__##__name */ - -struct DynLoad__cudnnGetErrorString { - template - const char* operator()(Args... args) { - return cudnnGetErrorString(args...); - } -} cudnnGetErrorString; /* struct DynLoad__##__name */ - #endif /** @@ -128,7 +96,9 @@ struct DynLoad__cudnnGetErrorString { __macro(cudnnPoolingForward) \ __macro(cudnnPoolingBackward) \ __macro(cudnnSoftmaxBackward) \ - __macro(cudnnSoftmaxForward) + __macro(cudnnSoftmaxForward) \ + __macro(cudnnGetVersion) \ + __macro(cudnnGetErrorString) CUDNN_DNN_ROUTINE_EACH(DYNAMIC_LOAD_CUDNN_WRAP) #define CUDNN_DNN_ROUTINE_EACH_R2(__macro) \ @@ -242,7 +212,7 @@ void hl_conv_workspace(hl_tensor_descriptor input, CHECK_NOTNULL(conv); // Specify workspace limit directly - size_t memoryLimitBytes = 8 * 1024 * 1024; + size_t memoryLimitBytes = (1LL << 20) * FLAGS_cudnn_conv_workspace_limit_in_mb; // cudnn convolution forward configuration cudnnTensorDescriptor_t fwd_src_desc = GET_TENSOR_DESCRIPTOR(input); @@ -340,7 +310,7 @@ void hl_create_tensor_descriptor(hl_tensor_descriptor* image_desc, (cudnn_tensor_descriptor)malloc(sizeof(_cudnn_tensor_descriptor)); CHECK_NOTNULL(hl_desc); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -373,7 +343,7 @@ void hl_create_tensor_descriptor(hl_tensor_descriptor* image_desc) { (cudnn_tensor_descriptor)malloc(sizeof(_cudnn_tensor_descriptor)); CHECK_NOTNULL(hl_desc); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -611,7 +581,7 @@ void hl_create_filter_descriptor(hl_filter_descriptor* filter, CHECK_CUDNN(dynload::cudnnCreateFilterDescriptor(&hl_filter->desc)); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -921,7 +891,7 @@ void hl_softmax_forward(real *input, int height, int width) { -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -955,7 +925,7 @@ void hl_softmax_backward(real *output_value, int height, int width) { -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index acd8e2fe6afb4..3ea2c91bd5a41 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -85,44 +85,24 @@ void* cudart_dso_handle = nullptr; #define DYNAMIC_LOAD_CUDART_WRAP(__name) \ struct DynLoad__##__name { \ template \ - cudaError_t operator()(Args... args) { \ - typedef cudaError_t (*cudartFunc)(Args...); \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using cudart_func = decltype(__name(args...))(*)(Args...); \ std::call_once(cudart_dso_flag, GetCudartDsoHandle, \ &cudart_dso_handle); \ void* p_##__name = dlsym(cudart_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ + return reinterpret_cast(p_##__name)(args...); \ } \ } __name; /* struct DynLoad__##__name */ #else #define DYNAMIC_LOAD_CUDART_WRAP(__name) \ struct DynLoad__##__name { \ template \ - cudaError_t operator()(Args... args) { \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ return __name(args...); \ } \ } __name; /* struct DynLoad__##__name */ #endif -#ifdef PADDLE_USE_DSO - struct DynLoad__cudaGetErrorString { - template - const char* operator()(Args... args) { - typedef const char* (*cudaFunc)(Args...); - std::call_once(cudart_dso_flag, GetCudartDsoHandle, - &cudart_dso_handle); - void* p_func = dlsym(cudart_dso_handle, "cudaGetErrorString"); - return reinterpret_cast(p_func)(args...); - } - } cudaGetErrorString; /* struct DynLoad__cudaGetErrorString */ -#else -struct DynLoad__cudaGetErrorString { - template - const char* operator()(Args... args) { - return cudaGetErrorString(args...); - } -} cudaGetErrorString; /* struct DynLoad__cudaGetErrorString */ -#endif - /* include all needed cuda functions in HPPL */ #define CUDA_ROUTINE_EACH(__macro) \ __macro(cudaMalloc) \ @@ -152,7 +132,8 @@ struct DynLoad__cudaGetErrorString { __macro(cudaSetDeviceFlags) \ __macro(cudaGetLastError) \ __macro(cudaFuncSetCacheConfig) \ - __macro(cudaRuntimeGetVersion) + __macro(cudaRuntimeGetVersion) \ + __macro(cudaGetErrorString) CUDA_ROUTINE_EACH(DYNAMIC_LOAD_CUDART_WRAP) @@ -211,7 +192,11 @@ bool hl_start_flag = false; inline pid_t gettid() { #if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); + // syscall is deprecated: first deprecated in macOS 10.12. + // syscall is unsupported; + // syscall pid_t tid = syscall(SYS_thread_selfid); + uint64_t tid; + pthread_threadid_np(NULL, &tid); #else #ifndef __NR_gettid #define __NR_gettid 224 @@ -622,7 +607,7 @@ void hl_specify_devices_start(int* device, int number) { void hl_rand(real *dest_d, size_t num) { pthread_mutex_lock(t_resource.gen_mutex); CHECK_EQ( -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE dynload::curandGenerateUniform(t_resource.gen, dest_d, num), #else dynload::curandGenerateUniformDouble(t_resource.gen, dest_d, num), diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index 38e4f16217c2a..0b7cd3375671d 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -18,8 +18,10 @@ limitations under the License. */ #include "hl_matrix_ops.cuh" #include "hl_matrix_apply.cuh" #include "hl_sequence.h" +#include "hl_sparse.ph" #include "paddle/utils/Logging.h" #include "hl_device_functions.cuh" +#include "hl_gpu_matrix_kernel.cuh" DEFINE_MATRIX_UNARY_OP(Zero, a = 0); DEFINE_MATRIX_TERNARY_PARAMETER_OP(_add, TWO_PARAMETER, c = p1*a + p2*b); @@ -47,7 +49,7 @@ void hl_matrix_add(real *A_d, CHECK_SYNC("hl_matrix_add failed"); } -#ifdef HPPL_TYPE_DOUBLE +#ifdef PADDLE_TYPE_DOUBLE #define THRESHOLD 128 #else #define THRESHOLD 64 @@ -102,7 +104,7 @@ void subMaxAndExp(real* I, val = -THRESHOLD; } I[nextIdx] = val; -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE O[nextIdx] = __expf(val); #else O[nextIdx] = exp(val); @@ -316,6 +318,85 @@ void hl_matrix_classification_error(real* A_d, CHECK_SYNC("hl_matrix_classification_error"); } +__global__ void KeMatrixMultiBinaryCrossEntropy(real* output, + real* entropy, + int* row, + int* col, + int dimM, + int dimN) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < dimM) { + for (int i = 0; i < dimN; i ++) { + entropy[index] -= log(1 - output[index * dimN + i]); + } + int *row_col = col + row[index]; + int col_num = row[index + 1] - row[index]; + for (int i = 0; i < col_num; i ++) { + real o = output[index * dimN + row_col[i]]; + entropy[index] -= log(o / (1 - o)); + } + } +} + +void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s csr_mat, + int dimM, + int dimN) { + CHECK_NOTNULL(output); + CHECK_NOTNULL(entropy); + CHECK_NOTNULL(csr_mat); + CHECK_EQ(csr_mat->format, HL_SPARSE_CSR); + int n_threads = 1024; + int blocks = (dimM + n_threads - 1) / n_threads; + dim3 threads(n_threads); + dim3 grid(blocks); + hl_csr_matrix mat = (hl_csr_matrix)(csr_mat->matrix); + KeMatrixMultiBinaryCrossEntropy<<< grid, threads, 0, STREAM_DEFAULT >>> + (output, entropy, mat->csr_row, mat->csr_col, dimM, dimN); + CHECK_SYNC("hl_matrix_multi_binary_cross_entropy failed"); +} + +__global__ void KeMatrixMultiBinaryCrossEntropyBp(real* output, + real* grad, + int* row, + int* col, + int dimM, + int dimN) { + int row_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (row_idx < dimM) { + for (int i = 0; i < dimN; i ++) { + int index = row_idx * dimN + i; + grad[index] += 1.0 / (1 - output[index]); + } + int col_num = row[row_idx + 1] - row[row_idx]; + int *row_col = col + row[row_idx]; + for (int i = 0; i < col_num; i ++) { + int index = row_idx * dimN + row_col[i]; + grad[index] -= 1.0 / (output[index] * (1 - output[index])); + } + } +} + +void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s csr_mat, + int dimM, + int dimN) { + CHECK_NOTNULL(output); + CHECK_NOTNULL(grad); + CHECK_NOTNULL(csr_mat); + CHECK_EQ(csr_mat->format, HL_SPARSE_CSR); + int n_threads = 1024; + int blocks = (dimM + n_threads - 1) / n_threads; + dim3 threads(n_threads); + dim3 grid(blocks); + hl_csr_matrix mat = (hl_csr_matrix)(csr_mat->matrix); + KeMatrixMultiBinaryCrossEntropyBp<<< grid, threads, 0, STREAM_DEFAULT >>> + (output, grad, mat->csr_row, mat->csr_col, dimM, dimN); + CHECK_SYNC("hl_matrix_multi_binary_cross_entropy_bp failed"); +} + __global__ void KeMatrixCrossEntropy(real* O, real* E, int* label, @@ -673,3 +754,89 @@ void hl_cossim_derivative(real* grad, input1_height, input2_height, scale); CHECK_SYNC("hl_cossim_derivate failed"); } + +__global__ void KeMatrixAddSharedBias(real* A, + real* B, + const int channel, + const int M, + const int N, + real scale) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + int dim = N / channel; + if (index < M * N) { + int i = index % N; + i = i / dim; + A[index] += scale * B[i]; + } +} + +void hl_matrix_add_shared_bias(real* A_d, + real* B_d, + const int channel, + const int dimM, + const int dimN, + real scale) { + const int blocks = 512; + const int grids = DIVUP(dimM * dimN, blocks); + KeMatrixAddSharedBias<<>> + (A_d, B_d, channel, dimM, dimN, scale); + CHECK_SYNC("hl_matrix_add_shared_bias failed"); +} + + +template +__global__ void KeMatrixCollectSharedBias(real *B, + real *A, + const int channel, + const int M, + const int N, + const int dim, + const int limit, + real scale) { + if (dim < limit) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < channel) { + real sum = 0.0; + for (int i = 0; i < M; ++i) { + for (int j = 0; j < dim; ++j) { + sum += A[i * N + index * dim + j]; + } + } + B[index] += scale * sum; + } + } else { + const int tid = threadIdx.x; + const int bid = blockIdx.x; + __shared__ real smem[blockSize]; + real sum = 0.0; + for (int j = 0; j < ((dim * M + blockSize - 1) / blockSize); ++j) { + int n = j * blockSize + tid; + int m = n / dim; + int w = n % dim; + smem[tid] = (m < M && w < dim) ? A[m * N + bid * dim + w] : 0.0; + __syncthreads(); + simpleReduce(smem, tid, blockSize); + sum += smem[0]; + } + if (tid == 0) { + B[bid] += scale * sum; + } + } +} + +void hl_matrix_collect_shared_bias(real* B_d, + real* A_d, + const int channel, + const int dimM, + const int dimN, + real scale) { + const int dim = dimN / channel; + const int blocks = 256; + const int limit = 64; + int grids = (dimM * dim) < limit ? DIVUP(channel, blocks) : channel; + + KeMatrixCollectSharedBias + <<< grids, blocks, 0, STREAM_DEFAULT>>> + (B_d, A_d, channel, dimM, dimN, dim, limit, scale); + CHECK_SYNC("hl_matrix_collect_shared_bias failed"); +} diff --git a/paddle/cuda/src/hl_cuda_sequence.cu b/paddle/cuda/src/hl_cuda_sequence.cu index e028880156e5b..63824eaa4c201 100644 --- a/paddle/cuda/src/hl_cuda_sequence.cu +++ b/paddle/cuda/src/hl_cuda_sequence.cu @@ -374,7 +374,7 @@ template __global__ void KeSequence2Batch(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount) { int idx = threadIdx.x; @@ -405,7 +405,7 @@ void KeSequence2Batch(real *batch, void hl_sequence2batch_copy(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount, bool seq2batch) { diff --git a/paddle/cuda/src/hl_cuda_sparse.cu b/paddle/cuda/src/hl_cuda_sparse.cu index b42568afdaaf5..1687fcc221ab8 100644 --- a/paddle/cuda/src/hl_cuda_sparse.cu +++ b/paddle/cuda/src/hl_cuda_sparse.cu @@ -562,6 +562,22 @@ void hl_memcpy_sparse_matrix(hl_sparse_matrix_s dst, } } +/** + * Calculate beta * C, if beta is zero, C does not have to be a valid input. + */ +static void _beta_mul_c(real *c, int dimM, int dimN, real beta) { + if (beta == 0.0) { + hl_gpu_apply_unary_op(unary::Zero(), c, dimM, dimN, dimN); + } else { + if (beta != 1.0){ + hl_gpu_apply_unary_op( + unary::mul_scalar(beta), c, dimM, dimN, dimN); + } + } + + return; +} + void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, real *B_d, hl_trans_op_t transb, real *C_d, @@ -580,15 +596,8 @@ void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, } if (A_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -633,13 +642,7 @@ void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, beta); } } else if (HPPL_OP_T == transa) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = (dimN + CU_CSC_MUL_DENSE_BLOCK_N - 1) / CU_CSC_MUL_DENSE_BLOCK_N; @@ -699,15 +702,8 @@ void hl_matrix_dense_mul_csc(real *A_d, hl_trans_op_t transa, << "matrix format error!"; if (B_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -750,13 +746,7 @@ void hl_matrix_dense_mul_csc(real *A_d, hl_trans_op_t transa, beta); } } else if (transb == HPPL_OP_T) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = 1 + (dimK-1)/CU_DM_CSR_THREAD_X; int blocksY = 1 + (dimM-1)/CU_DM_CSR_BLOCK_M; dim3 threads(CU_DM_CSR_THREAD_X, CU_DM_CSR_THREAD_Y); @@ -813,15 +803,8 @@ void hl_matrix_dense_mul_csr(real *A_d, hl_trans_op_t transa, << "matrix format error!"; if (B_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -833,14 +816,7 @@ void hl_matrix_dense_mul_csr(real *A_d, hl_trans_op_t transa, } if (transb == HPPL_OP_N) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } - + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = 1 + (dimK-1)/CU_DM_CSR_THREAD_X; int blocksY = 1 + (dimM-1)/CU_DM_CSR_BLOCK_M; dim3 threads(CU_DM_CSR_THREAD_X, CU_DM_CSR_THREAD_Y); @@ -925,15 +901,8 @@ void hl_matrix_csc_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, } if (A_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -945,13 +914,7 @@ void hl_matrix_csc_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, } if (HPPL_OP_N == transa) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = (dimN + CU_CSC_MUL_DENSE_BLOCK_N -1)/CU_CSC_MUL_DENSE_BLOCK_N; int blocksY = (dimK + CU_CSC_MUL_DENSE_BLOCK_K -1)/CU_CSC_MUL_DENSE_BLOCK_K; @@ -1113,7 +1076,7 @@ void hl_sparse_matrix_mul(real *A_d, hl_trans_op_t transa, CHECK(!transA) << "Not supported A is trans and B is not trans!"; dim3 block(CU_BLOCK_SIZE, 1); - int avgNnzPerRow = C_d2->nnz_s / dimM; + int avgNnzPerRow = C_d->nnz / dimM; avgNnzPerRow = avgNnzPerRow > 0 ? avgNnzPerRow : 1; int gridx = DIVUP(avgNnzPerRow, CU_BLOCK_SIZE); dim3 grid(gridx, dimM); @@ -1242,9 +1205,9 @@ void hl_matrix_csr_column_sum(real* A_d, hl_sparse_matrix_s B_d, LOG(FATAL) << "parameter B is null!"; } - if (B_d2->nnz_s == 0) return; + if (B_d->nnz == 0) return; - int nnz = B_d2->nnz_s; + int nnz = B_d->nnz; int block = 512; int grid = DIVUP(nnz, 512); KeSMatrixCsrColumnSum<<>>( @@ -1273,9 +1236,9 @@ void hl_matrix_csr_add_bias(hl_sparse_matrix_s A_d, real* B_d, LOG(FATAL) << "parameter A_d is null!"; } - if (A_d2->nnz_s == 0) return; + if (A_d->nnz == 0) return; - int nnz = A_d2->nnz_s; + int nnz = A_d->nnz; int block = 512; int grid = DIVUP(nnz, 512); KeSMatrixCsrAddBias<<>>( @@ -1308,9 +1271,9 @@ void hl_matrix_csr_add_dense(hl_sparse_matrix_s A_d, real* B_d, int dimM, LOG(FATAL) << "parameter A_d is null!"; } - if (A_d2->nnz_s == 0) return; + if (A_d->nnz == 0) return; - int gridX = DIVUP((A_d2->nnz_s / dimM), 512); + int gridX = DIVUP((A_d->nnz / dimM), 512); gridX = gridX > 0 ? gridX : 1; dim3 block(512, 1); dim3 grid(gridX, dimM); diff --git a/paddle/cuda/src/hl_cuda_sparse.cuh b/paddle/cuda/src/hl_cuda_sparse.cuh index db5c9ce979885..9cf2d5a843343 100644 --- a/paddle/cuda/src/hl_cuda_sparse.cuh +++ b/paddle/cuda/src/hl_cuda_sparse.cuh @@ -85,6 +85,15 @@ __global__ void KeSMatrixCsc2Dense(real * csc_val, C_d[row*dimN + col] = sum; } +__device__ __forceinline__ +void _calculate_c(real &c, real sum) { + c = sum; +} +__device__ __forceinline__ +void _calculate_c(real &c, real sum, real beta) { + c = sum + beta * c; +} + #define CU_CSRMM_N 4 #define CU_CSRMM_THREAD_X 32 #define CU_CSRMM_THREAD_Y 32 @@ -191,11 +200,19 @@ __global__ void KeSMatrixCsrMulDense(real *C_d, } C_d += __mul24(index_m, dimN); - #pragma unroll - for (int n = 0; n < CU_CSRMM_N; n++) { - if (index_n < dimN) { - C_d[index_n] = alpha*sum[n] + beta*C_d[index_n]; - index_n += CU_CSRMM_THREAD_X; + if (beta == 0.0) { + for (int n = 0; n < CU_CSRMM_N; n++) { + if (index_n < dimN) { + _calculate_c(C_d[index_n], alpha * sum[n]); + index_n += CU_CSRMM_THREAD_X; + } + } + } else { + for (int n = 0; n < CU_CSRMM_N; n++) { + if (index_n < dimN) { + _calculate_c(C_d[index_n], alpha * sum[n], beta); + index_n += CU_CSRMM_THREAD_X; + } } } } @@ -338,7 +355,7 @@ __global__ void KeSMatrixCscMulDense(real *C_d, } /* best perf */ -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define CU_CSCMM_THREAD_M_BEST 9 #else #define CU_CSCMM_THREAD_M_BEST 4 @@ -544,13 +561,22 @@ TEMP_TEST: int index_m_c = ibx + idy; int index_n_c = blockIdx.y*CU_CSCMM_BLOCK_N_BEST + idx; C_d += index_n_c + __mul24(index_m_c, dimN); - #pragma unroll - for (int m = 0; m < CU_CSCMM_THREAD_M_BEST; m++) { - if (index_m_c < dimM && index_n_c < dimN) { - C_d[0] = A_s[idy+m*32][idx] + beta*C_d[0]; + if (beta == 0.0) { + for (int m = 0; m < CU_CSCMM_THREAD_M_BEST; m++) { + if (index_m_c < dimM && index_n_c < dimN) { + _calculate_c(C_d[0], A_s[idy + m * 32][idx]); + } + index_m_c += 32; + C_d += dimN*32; + } + } else { + for (int m = 0; m < CU_CSCMM_THREAD_M_BEST; m++) { + if (index_m_c < dimM && index_n_c < dimN) { + _calculate_c(C_d[0], A_s[idy + m * 32][idx], beta); + } + index_m_c += 32; + C_d += dimN*32; } - index_m_c += 32; - C_d += dimN*32; } } @@ -882,24 +908,6 @@ int findIndex(int* indice, int num, int index) { return (end - 1); } -/** - * @brief sum reduction - * - * @param[in,out] smem input data, better to use __shared__ memory. - * @param[in] tid local thread index. - * @param[in] blockDimX the size of blockDim.x. - * - * note: return smem[0]: the sum of each elements of smem. - */ -__device__ __forceinline__ -void reduce(real* smem, int tid, int blockDimX) { - for (unsigned int s = blockDimX / 2; s > 0; s >>= 1) { - if (tid < s) { - smem[tid] += smem[tid + s]; - } - __syncthreads(); - } -} /** * @brief sum columns of csr sparse matrix (csr_val), then add to a_val. diff --git a/paddle/cuda/src/hl_dso_loader.cc b/paddle/cuda/src/hl_dso_loader.cc index eee9984e07326..b564b96903368 100644 --- a/paddle/cuda/src/hl_dso_loader.cc +++ b/paddle/cuda/src/hl_dso_loader.cc @@ -46,63 +46,100 @@ static inline std::string join(const std::string& part1, const std::string& part return ret; } -static inline void GetDsoHandleWithSearchPath( +static inline void GetDsoHandleFromDefaultPath( + std::string& dso_path, void** dso_handle, int dynload_flags) { + VLOG(3) << "Try to find cuda library: " << dso_path + << " from default system path."; + // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH + *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + + // DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to + // bring System Integrity Projection (SIP), if dso_handle + // is null, search from default package path in Mac OS. + #if defined(__APPLE__) || defined(__OSX__) + if (nullptr == *dso_handle) { + dso_path = join("/usr/local/cuda/lib/", dso_path); + *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + if (nullptr == *dso_handle) { + if (dso_path == "libcudnn.dylib") { + LOG(FATAL) << "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n" + << "For instance, sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C " + << "/usr/local \n sudo chmod a+r /usr/local/cuda/include/cudnn.h " + << "/usr/local/cuda/lib/libcudnn*"; + } + } + } + #endif +} + +static inline void GetDsoHandleFromSearchPath( const std::string& search_root, - const std::string& dso_path, + const std::string& dso_name, void** dso_handle) { int dynload_flags = RTLD_LAZY | RTLD_LOCAL; *dso_handle = nullptr; - std::string dlPath = dso_path; + std::string dlPath = dso_name; if (search_root.empty()) { - // default search xxx.so from LD_LIBRARY_PATH - *dso_handle = dlopen(dlPath.c_str(), dynload_flags); + GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); } else { // search xxx.so from custom path - dlPath = join(search_root, dso_path); + dlPath = join(search_root, dso_name); *dso_handle = dlopen(dlPath.c_str(), dynload_flags); - // then, search xxx.so from LD_LIBRARY_PATH - if (nullptr == *dso_handle) { - *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + // if not found, search from default path + if (nullptr == dso_handle) { + LOG(WARNING) << "Failed to find cuda library: " << dlPath; + dlPath = dso_name; + GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); } } CHECK(nullptr != *dso_handle) - << "For Gpu version of PaddlePaddle, it couldn't find CUDA library: " - << dlPath.c_str() << ". Please make sure you already specify its path. " - << "Note: for training data on Cpu using Gpu version of PaddlePaddle, " - << "you must specify libcudart via export LD_LIBRARY_PATH for Linux or " - << "export DYLD_LIBRARY_PATH for MAC OS."; + << "Failed to find cuda library: " << dlPath << std::endl + << "Please specify its path correctly using one of the following ideas: \n" + + << "Idea 1. set cuda and cudnn lib path at runtime. " + << "http://www.paddlepaddle.org/doc/ui/cmd_argument/argument_outline.html \n" + << "For instance, issue command: paddle train --use_gpu=1 " + << "--cuda_dir=/usr/local/cudnn/lib --cudnn_dir=/usr/local/cudnn/lib ...\n" + + << "Idea 2. set environment variable LD_LIBRARY_PATH on Linux or " + << "DYLD_LIBRARY_PATH on Mac OS. \n" + << "For instance, issue command: export LD_LIBRARY_PATH=... \n" + + << "Note: After Mac OS 10.11, using the DYLD_LIBRARY_PATH is impossible " + << "unless System Integrity Protection (SIP) is disabled. However, @Idea 1" + << "always work well."; } void GetCublasDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle); #else - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle); #endif } void GetCudnnDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle); #else - GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle); #endif } void GetCudartDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath("", "libcudart.dylib", dso_handle); + GetDsoHandleFromSearchPath("", "libcudart.dylib", dso_handle); #else - GetDsoHandleWithSearchPath("", "libcudart.so", dso_handle); + GetDsoHandleFromSearchPath("", "libcudart.so", dso_handle); #endif } void GetCurandDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle); #else - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle); #endif } diff --git a/paddle/gserver/CMakeLists.txt b/paddle/gserver/CMakeLists.txt index 9ac4d210f6d37..a066f80c221ee 100644 --- a/paddle/gserver/CMakeLists.txt +++ b/paddle/gserver/CMakeLists.txt @@ -50,7 +50,7 @@ if(NOT WITH_PYTHON) endif() if(WITH_GPU) - add_paddle_culib(paddle_gserver ${GSERVER_SOURCES}) + cuda_add_library(paddle_gserver ${GSERVER_SOURCES}) else() add_library(paddle_gserver STATIC ${GSERVER_SOURCES}) diff --git a/paddle/gserver/activations/ActivationFunction.cpp b/paddle/gserver/activations/ActivationFunction.cpp index 9918d20d9082a..27eed75d4d76c 100644 --- a/paddle/gserver/activations/ActivationFunction.cpp +++ b/paddle/gserver/activations/ActivationFunction.cpp @@ -295,6 +295,7 @@ void forward(Argument& act) { void backward(Argument& act) { act.grad->squareDerivative(*act.in); } END_DEFINE_ACTIVATION(square) + /** * @brief Exponential Activation. * \f[ @@ -307,8 +308,36 @@ void forward(Argument& act) { act.value->exp(*act.value); } void backward(Argument& act) { act.grad->expDerivative(*act.value); } END_DEFINE_ACTIVATION(exponential) +/** + * @brief Logarithm Activation. + * \f[ + * f(z) = log(z) + * \f] + */ +BEGIN_DEFINE_ACTIVATION(log) +void forward(Argument& act) { + SetDevice device(act.deviceId); + Matrix::resizeOrCreate(act.in, act.value->getHeight(), act.value->getWidth(), + /* trans */ false, useGpu(act.deviceId)); + + act.in->copyFrom(*act.value); + act.value->log(*act.value); +} + +void backward(Argument& act) { act.grad->dotDiv(*act.grad, *act.in); } +END_DEFINE_ACTIVATION(log) + ActivationFunction* ActivationFunction::create(const std::string& type) { return gActivationRegistrar.createByType(type); } +std::vector ActivationFunction::getAllRegisteredTypes() { + std::vector types; + gActivationRegistrar.forEachType([&](const std::string& type) { + types.push_back(type); + }); + return types; +} + + } // namespace paddle diff --git a/paddle/gserver/activations/ActivationFunction.h b/paddle/gserver/activations/ActivationFunction.h index 29860b4a736c3..c483372256c03 100644 --- a/paddle/gserver/activations/ActivationFunction.h +++ b/paddle/gserver/activations/ActivationFunction.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include namespace paddle { @@ -32,6 +33,7 @@ struct Argument; class ActivationFunction { public: static ActivationFunction* create(const std::string& type); + static std::vector getAllRegisteredTypes(); ActivationFunction() {} diff --git a/paddle/gserver/dataproviders/DataProvider.cpp b/paddle/gserver/dataproviders/DataProvider.cpp index c3b4769f7612b..2cfb5a3a18c8a 100644 --- a/paddle/gserver/dataproviders/DataProvider.cpp +++ b/paddle/gserver/dataproviders/DataProvider.cpp @@ -57,7 +57,8 @@ void BufferBatch::clone(DataBatch* srcBatch, bool useGpu) { } } -DoubleBuffer::DoubleBuffer(DataProvider* dataPool, bool useGpu, +DoubleBuffer::DoubleBuffer(DataProvider *dataPool, + bool useGpu, int64_t batchSize) { batchSize_ = batchSize; dataPool_ = dataPool; @@ -110,6 +111,9 @@ void DoubleBuffer::removeOneBatch(DataBatch* dataBatch) { } void DoubleBuffer::insertOneBatch(DataBatch* batch) { + while (!bufferQueue_->waitNotEmptyFor(2 /* seconds */)) { // time out + if (stopping_) return; + } BufferBatch* bufBatch = bufferQueue_->dequeue(); // clone and copy the data from an Threadlocal Variable bufBatch->clone(batch, useGpu_); @@ -127,9 +131,10 @@ void DoubleBuffer::asyncLoadBatch() { taskReadySem_.wait(); if (stopping_) break; - while (batchSize_ == 0) { + while (batchSize_ == 0 && !stopping_) { usleep(5); } + if (stopping_) break; do { DataBatch newBatch; @@ -138,7 +143,7 @@ void DoubleBuffer::asyncLoadBatch() { actualSize = dataPool_->getNextBatchInternal(batchSize_, &newBatch); } insertOneBatch(&newBatch); - } while (actualSize > 0); + } while (actualSize > 0 && !stopping_); } } diff --git a/paddle/gserver/dataproviders/DataProvider.h b/paddle/gserver/dataproviders/DataProvider.h index 534491d70d546..112e45de1cb23 100644 --- a/paddle/gserver/dataproviders/DataProvider.h +++ b/paddle/gserver/dataproviders/DataProvider.h @@ -259,7 +259,9 @@ typedef Queue BufferBatchQueue; class DoubleBuffer { public: - DoubleBuffer(DataProvider* dataPool, bool useGpu, int64_t batchSize = 0); + DoubleBuffer(DataProvider* dataPool, + bool useGpu, + int64_t batchSize = 0); virtual ~DoubleBuffer(); void removeOneBatch(DataBatch* dataBatch); @@ -308,7 +310,8 @@ class DataProvider { /** * @brief create only used for unittest. */ - inline static DataProvider* create(const DataConfig &config, bool useGpu) { + inline static DataProvider* create(const DataConfig &config, + bool useGpu = FLAGS_use_gpu) { return create(config, ModelConfig(), useGpu); } @@ -348,7 +351,6 @@ class DataProvider { */ virtual void reset() { if (doubleBuffer_ != nullptr) { - LOG(INFO) << "the double-buffer is starting ..."; doubleBuffer_->startAsyncLoad(); } } diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index 2f9a1223c6e45..90391a7c307d8 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -14,13 +14,20 @@ limitations under the License. */ #ifndef PADDLE_NO_PYTHON +#include #include #include #include #include +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include #include "DataProvider.h" + #include "paddle/utils/PythonUtil.h" +#include "paddle/utils/Locks.h" +#include "paddle/utils/Stat.h" namespace paddle { @@ -202,7 +209,10 @@ class PyDataProvider2 : public DataProvider { PyDataProvider2(const DataConfig& config, const ModelConfig& modelConfig, bool useGpu) - :DataProvider(config, useGpu), callingContextCreated_(2) { + :DataProvider(config, useGpu), + callingContextCreated_(2) { + if (PyArray_API == NULL) + import_array(); auto& args = config.load_data_args(); PyObjectPtr kwargs = PyObjectPtr(PyDict_New()); if (!args.empty()) { @@ -246,8 +256,7 @@ class PyDataProvider2 : public DataProvider { PyObjectPtr && kwargs) { LOG(INFO) << "loading dataprovider " << model <<"::" << className; - PyObjectPtr module(PyImport_ImportModule(model.c_str())); - CHECK_PY(module) << "Cannot imort module " << model.c_str(); + PyObjectPtr module = py::import(model); PyObjectPtr moduleDict(PyModule_GetDict(module.get())); CHECK_PY(moduleDict) << "Invoke module.__dict__ error"; PyObjectPtr cls(PyDict_GetItemString(moduleDict.get(), @@ -424,26 +433,34 @@ class PyDataProvider2 : public DataProvider { inline void resetImpl(bool startNewThread) { DBG << "Reseting " << startNewThread; + exit_.store(true); if (loadThread_) { // is loading. - exit_.store(true); loadThread_->join(); loadThread_.reset(); } { PyGuard g; callingContexts_.clear(); + this->pullCV_.notify_one(); + } + + std::lock_guard guard(mutexForReset_); + { + PyGuard g; dataPool_.clear(); } poolActualSize_ = 0; - exit_ = false; + if (startNewThread && cache_->reset()) { DBG << "Start new thread."; loadThread_.reset(new std::thread([this] { + exit_ = false; loadThread(); })); callingContextCreated_.wait(); } DBG << "Reset done"; + exit_ = false; } private: @@ -455,6 +472,9 @@ class PyDataProvider2 : public DataProvider { std::condition_variable pushCV_; std::condition_variable pullCV_; std::mutex mtx_; + + std::mutex mutexForReset_; + ThreadBarrier callingContextCreated_; std::unique_ptr cache_; @@ -497,8 +517,8 @@ class PyDataProvider2 : public DataProvider { * Resetting the PyDataProvider. May start reading thread here. */ virtual void reset() { - DataProvider::reset(); resetImpl(true); + DataProvider::reset(); } /** @@ -519,6 +539,8 @@ class PyDataProvider2 : public DataProvider { * Loading a batch of data. */ int64_t getNextBatchInternal(int64_t size_, DataBatch *batch) { + std::lock_guard guard(mutexForReset_); + REGISTER_TIMER("PyDP2.getNextBatchInternal") CHECK_GE(size_, 0); size_t size = (size_t) size_; if (loadThread_) { // loading from thread should wait for data pool ready. @@ -543,6 +565,10 @@ class PyDataProvider2 : public DataProvider { } else { // loading from cache. poolPtr = this->cache_->load(); } + if (exit_) { + // PyDataProvider is destructing. + return 0; + } CHECK(poolPtr != nullptr); std::deque& pool = *poolPtr; @@ -699,10 +725,22 @@ class DenseScanner: public IFieldScanner { */ virtual void fill(Argument &argument, PyObject *obj) { real* dat = argument.value->getData() + height_ * headerPtr_->dim; - py::SequenceHelper s(obj); - // TODO(yuyang18): Here we can use AVX or SSE to accelerate memory copy. - for (size_t i=0; i < headerPtr_->dim; ++i) { - dat[i] = (real) s.getDouble(i); + if (PyArray_Check(obj)) { + auto dtype = PyArray_DTYPE((PyArrayObject*)obj); + if (dtype->type == 'f' && dtype->elsize == sizeof(real)) { + real * data = (real*)PyArray_DATA((PyArrayObject*)obj); + auto sz = PyArray_SIZE((PyArrayObject*)obj); + std::copy(data, data + sz, dat); + } else { + LOG(FATAL) << "You should yield float" << sizeof(real) * 8 + << " array"; + } + } else { + py::SequenceHelper s(obj); + // TODO(yuyang18): Here we can use AVX or SSE to accelerate memory copy. + for (size_t i=0; i < headerPtr_->dim; ++i) { + dat[i] = (real) s.getDouble(i); + } } ++height_; } diff --git a/paddle/gserver/evaluators/ChunkEvaluator.cpp b/paddle/gserver/evaluators/ChunkEvaluator.cpp index 273925ba55ee4..22579891f397a 100644 --- a/paddle/gserver/evaluators/ChunkEvaluator.cpp +++ b/paddle/gserver/evaluators/ChunkEvaluator.cpp @@ -75,7 +75,6 @@ class ChunkEvaluator : public Evaluator { public: virtual void init(const EvaluatorConfig& config) { - CHECK(!FLAGS_use_gpu) << "Not supported"; Evaluator::init(config); if (config.chunk_scheme() == "IOB") { numTagTypes_ = 2; @@ -137,6 +136,7 @@ class ChunkEvaluator : public Evaluator { CHECK_EQ(arguments.size(), (size_t)2); IVectorPtr& output = arguments[0].ids; IVectorPtr& label = arguments[1].ids; + CHECK(!output->useGpu() && !label->useGpu()) << "Not supported"; auto sequenceStartPositions = arguments[1].sequenceStartPositions->getVector(false); CHECK_EQ(output->getSize(), label->getSize()); diff --git a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp index 787ce703a08ae..0ded30eeb44e9 100644 --- a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp @@ -813,7 +813,6 @@ void TrainerThread::mergeGradSparse( para->getMat(PARAMETER_GRADIENT).get()); std::vector& ids = mainMat->getIds(threadId_); - ids.clear(); for (auto slaveParams : slaveParameters) { SparseRowCpuMatrix* mat = dynamic_cast((*slaveParams)[pid] diff --git a/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp b/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp index 952df60a7d786..22698f5867017 100644 --- a/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp @@ -28,6 +28,12 @@ void ParallelNeuralNetwork::init( const std::vector& parameterTypes, bool useGpu) { NeuralNetwork::init(config, callback, parameterTypes, useGpu); + if (config.type() == "recurrent_nn") { + LOG(FATAL) + << "You can not add `--parallel_nn=true` on the command line, " + << "parallel_nn training mode does not support the recurrent_nn model."; + } + useGpu_ = useGpu; numDevices_ = 0; if (useGpu_) { diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index fc38bca3c403b..340cd1b9f8e92 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -544,6 +544,12 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, const std::vector inArgs; std::vector outArgs; frames_[i]->forward(inArgs, &outArgs, passType); + if (hasSubseq) { + for (auto& outFrameLine : outFrameLines_) { + CHECK(outFrameLine.frames[i]->getOutput().sequenceStartPositions) + << "In hierachical RNN, all out links should be from sequences."; + } + } } if (evaluator_ && passType == PASS_TEST) { this->eval(evaluator_.get()); @@ -635,16 +641,15 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinkId, std::vector sequenceStartPositions; const int* subSequenceStartPositions = nullptr; - if (hasSubseq) { // for sequenceScatterAgentLayer - subSequenceStartPositions = - input.subSequenceStartPositions->getData(false); + if (hasSubseq) { // for sequenceScatterAgentLayer + subSequenceStartPositions = input.subSequenceStartPositions->getData(false); inlinkInfo->seqStartPosIndex.clear(); inlinkInfo->seqStartPosIndex.push_back(0); // first seqStartPosIndex = 0 } // maxSequenceLength_: max topLevelLength in allsamples for (int i = 0; i < maxSequenceLength_; ++i) { if (hasSubseq) { - sequenceStartPositions.push_back(0); // first element = 0 + sequenceStartPositions.push_back(0); // first element = 0 } int numSeqs = 0; for (size_t j = 0; j < numSequences; ++j) { @@ -676,9 +681,9 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinkId, } if (hasSubseq) { // inFrameLine create sequenceStartPositions one time - CHECK_EQ(sequenceStartPositions.size(), - static_cast(maxSequenceLength_ + - input.getNumSubSequences())); + CHECK_EQ( + sequenceStartPositions.size(), + static_cast(maxSequenceLength_ + input.getNumSubSequences())); CHECK_EQ(inlinkInfo->seqStartPosIndex.size(), static_cast(maxSequenceLength_ + 1)); createSeqPos(sequenceStartPositions, &inlinkInfo->sequenceStartPositions); @@ -1102,10 +1107,12 @@ size_t RecurrentGradientMachine::beamShrink(std::vector& newPaths, newPaths.end(), Path::greaterPath); newPaths.resize(totalExpandCount + minNewPathSize); - real minPathLogProb = std::min_element(newPaths.end() - minNewPathSize, - newPaths.end())->logProb; - real maxPathLogProb = std::max_element(newPaths.end() - minNewPathSize, - newPaths.end())->logProb; + real minPathLogProb = + std::min_element(newPaths.end() - minNewPathSize, newPaths.end()) + ->logProb; + real maxPathLogProb = + std::max_element(newPaths.end() - minNewPathSize, newPaths.end()) + ->logProb; // Remove the already formed paths that are relatively short finalPaths_[seqId].erase( diff --git a/paddle/gserver/layers/AgentLayer.cpp b/paddle/gserver/layers/AgentLayer.cpp index 056e9568852ac..5e07446c71ff6 100644 --- a/paddle/gserver/layers/AgentLayer.cpp +++ b/paddle/gserver/layers/AgentLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "AgentLayer.h" #include "paddle/utils/Logging.h" @@ -62,8 +61,8 @@ void SequenceAgentLayer::forward(PassType passType) { // get Arguments from real layers if (numSamples_ > 0 && numSamples_ < realNumSequences) { - int numRows = realOutput.sequenceStartPositions-> - getData(false)[numSamples_]; + int numRows = + realOutput.sequenceStartPositions->getData(false)[numSamples_]; CHECK(!realOutput.ids) << "Not supported"; output_.subArgFrom(realOutput, /* offset */ 0, numRows, getSize(), useGpu_, /* trans */ false, /* seqFlag */ true, @@ -141,8 +140,8 @@ void ScatterAgentLayer::forward(PassType passType) { int width = this->getSize(); if (realOutArg_.value || realOutArg_.ids) { - output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, - width, useGpu_); + output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, width, + useGpu_); } else { // used in generation if (realLayer_->getOutput().ids) { IVector::resizeOrCreate(output_.ids, ids_->getSize(), useGpu_); @@ -224,8 +223,8 @@ void SequenceScatterAgentLayer::forward(PassType passType) { if (realOutArg_.value || realOutArg_.ids) { CHECK(realOutArg_.sequenceStartPositions); - output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, - width, useGpu_, /* trans */ false, /* seqFlag */ true, + output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, width, + useGpu_, /* trans */ false, /* seqFlag */ true, /* seqStart */ seqStartPosIndex_, /* seqSize */ numSequences_); } else { @@ -249,11 +248,12 @@ void SequenceScatterAgentLayer::forward(PassType passType) { CHECK_NE(input.sequenceStartPositions.get(), output_.sequenceStartPositions.get()); ICpuGpuVector::resizeOrCreate(output_.sequenceStartPositions, - numSequences + 1, false); + numSequences + 1, false); int* outStarts = output_.sequenceStartPositions->getMutableData(false); - IVector::resizeOrCreate(cpuInputStartPos_, height, false); - int* inStarts = cpuInputStartPos_->getData(); + ICpuGpuVector::resizeOrCreate(inputStartPos_, height, false); + int* inStarts = inputStartPos_->getMutableData(false); + size_t offsetOut = 0; for (size_t i = 0; i < numSequences; ++i) { outStarts[i] = offsetOut; @@ -266,13 +266,8 @@ void SequenceScatterAgentLayer::forward(PassType passType) { } outStarts[numSequences] = offsetOut; - if (useGpu_) { - IVector::resizeOrCreate(inputStartPos_, height, true); - inputStartPos_->copyFrom(*cpuInputStartPos_, HPPL_STREAM_DEFAULT); - } else { - inputStartPos_ = cpuInputStartPos_; - } - outputValue->copyByRowIndex(*input.value, *inputStartPos_); + outputValue->copyByRowIndex(*input.value, + *inputStartPos_->getVector(useGpu_)); } } diff --git a/paddle/gserver/layers/AgentLayer.h b/paddle/gserver/layers/AgentLayer.h index d82078dd93329..3d7bf55834070 100644 --- a/paddle/gserver/layers/AgentLayer.h +++ b/paddle/gserver/layers/AgentLayer.h @@ -191,11 +191,7 @@ class SequenceScatterAgentLayer : public ScatterAgentLayer { protected: // use to store expanded cpuStartPositions or subSequenceStartPositions // of real layer. - IVectorPtr cpuInputStartPos_; - - // point to cpuInputStartPos_ when useGpu_ is false - // copy from cpuInputStartPos_ when useGpu_ is true - IVectorPtr inputStartPos_; + ICpuGpuVectorPtr inputStartPos_; public: explicit SequenceScatterAgentLayer(const LayerConfig& config) diff --git a/paddle/gserver/layers/AverageLayer.cpp b/paddle/gserver/layers/AverageLayer.cpp index 374117b7659bb..7401cdc9a516b 100644 --- a/paddle/gserver/layers/AverageLayer.cpp +++ b/paddle/gserver/layers/AverageLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "AverageLayer.h" #include "paddle/utils/Logging.h" @@ -25,13 +24,8 @@ REGISTER_LAYER(average, AverageLayer); bool AverageLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { - /* Initialize the basic parent class */ - Layer::init(layerMap, parameterMap); + SequencePoolLayer::init(layerMap, parameterMap); - /* initialize biases_ */ - if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); - } dataMtx_ = Matrix::create(nullptr, 1, 1, false, useGpu_); outMtx_ = Matrix::create(nullptr, 1, getSize(), false, useGpu_); // average strategy @@ -44,57 +38,15 @@ bool AverageLayer::init(const LayerMap& layerMap, } else { LOG(FATAL) << "Unknown average strategy: " << config_.average_strategy(); } - // transform to which sequence type - if (config_.trans_type() == "non-seq") { - type_ = kNonSeq; - } else if (config_.trans_type() == "seq") { - type_ = kSeq; - } else { - LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); - } - setNeedSequenceInfo(false); return true; } void AverageLayer::forward(PassType passType) { - Layer::forward(passType); - - // average layer should have exactly 1 input - CHECK_EQ(1U, inputLayers_.size()); - - size_t dim = getSize(); - const Argument& input = getInput(0); - int64_t newBatchSize = - type_ ? input.getNumSubSequences() : input.getNumSequences(); - ICpuGpuVectorPtr startPositions = - type_ ? input.subSequenceStartPositions - : input.sequenceStartPositions; - const int* starts = startPositions->getData(false); - size_t numSequences = startPositions->getSize() - 1; - - // check - CHECK_EQ(numSequences, (size_t)newBatchSize); - CHECK_EQ(starts[numSequences], input.getBatchSize()); - if (type_) { - // when trans_type = seq, input must hasSubseq - CHECK_EQ(input.hasSubseq(), 1UL); - } + SequencePoolLayer::forward(passType); - CHECK_EQ(dim, input.value->getWidth()); - - resetOutput(newBatchSize, dim); - auto startsPos = startPositions->getVector(useGpu_); MatrixPtr inputValue = getInputValue(0); - getOutputValue()->sequenceAvgForward(*inputValue, *startsPos, mode_); - - /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, - * thus, in this case, output_ has no sequenceStartPositions. - * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this - * case, we should compute the new sequenceStartPositions. - */ - if (type_) { - output_.degradeSequence(input, useGpu_); - } + getOutputValue()->sequenceAvgForward( + *inputValue, *startPositions_->getVector(useGpu_), mode_); /* add the bias-vector AFTER average operation */ if (biases_.get() != NULL) { @@ -106,26 +58,16 @@ void AverageLayer::forward(PassType passType) { } void AverageLayer::backward(const UpdateCallback& callback) { - const Argument& input = getInput(0); - ICpuGpuVectorPtr startPositions = - type_ ? input.subSequenceStartPositions - : input.sequenceStartPositions; - const int* starts = startPositions->getData(false); - /* Do derivation */ { backwardActivation(); } - - if (biases_ && biases_->getWGrad()) { - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - - // Increasing the number of gradient - biases_->getParameterPtr()->incUpdate(callback); - } + SequencePoolLayer::backward(callback); + const int* starts = startPositions_->getData(false); MatrixPtr grad = getInputGrad(0); + if (grad) { size_t dim = getSize(); real* gradientData = getInputGrad(0)->getData(); real* gradient = getOutputGrad()->getData(); - size_t numSequences = startPositions->getSize() - 1; + size_t numSequences = startPositions_->getSize() - 1; for (size_t sequenceId = 0; sequenceId < numSequences; ++sequenceId) { // TODO(Dangqingqing) optimization for GPU int sequenceLength = starts[sequenceId + 1] - starts[sequenceId]; diff --git a/paddle/gserver/layers/AverageLayer.h b/paddle/gserver/layers/AverageLayer.h index ae910ddefad13..1edc2ace492c5 100644 --- a/paddle/gserver/layers/AverageLayer.h +++ b/paddle/gserver/layers/AverageLayer.h @@ -12,10 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once -#include "Layer.h" +#include "SequencePoolLayer.h" #include "paddle/math/Matrix.h" namespace paddle { @@ -23,20 +22,21 @@ namespace paddle { /** * A layer for "internal average" for sequence input. * Input: one or more sequences. Each sequence contains some instances. - * If AverageLevel = kNonSeq: + * If SequenceLevel = kNonSeq: * Output: output size is the number of input sequences (NOT input instances) * output[i] = average_{for each instance in this sequence}{input[i]} - * If AverageLevel = kSeq: + * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: output size is the number of input sub-sequences * output[i] = average_{for each instance in this sub-sequence}{input[i]} + * + * The config file api is pooling_layer. */ - -class AverageLayer : public Layer { +class AverageLayer : public SequencePoolLayer { public: enum AverageStrategy { kAverage = 0, kSum = 1, kAverageSquareRootN = 2 }; - enum AverageLevel { kNonSeq = 0, kSeq = 1 }; - explicit AverageLayer(const LayerConfig& config) : Layer(config) {} + explicit AverageLayer(const LayerConfig& config) + : SequencePoolLayer(config) {} ~AverageLayer() {} @@ -46,11 +46,8 @@ class AverageLayer : public Layer { void backward(const UpdateCallback& callback = nullptr); protected: - std::unique_ptr biases_; MatrixPtr outMtx_; MatrixPtr dataMtx_; int mode_; - int type_; }; - } // namespace paddle diff --git a/paddle/gserver/layers/BilinearInterpLayer.cpp b/paddle/gserver/layers/BilinearInterpLayer.cpp new file mode 100644 index 0000000000000..ac5f87be7af07 --- /dev/null +++ b/paddle/gserver/layers/BilinearInterpLayer.cpp @@ -0,0 +1,95 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "BilinearInterpLayer.h" +#include "paddle/utils/Logging.h" +#include "paddle/utils/Stat.h" + +namespace paddle { + +REGISTER_LAYER(bilinear_interp, BilinearInterpLayer); + +size_t BilinearInterpLayer::getSize() { + inImgH_ = inputLayers_[0]->getOutput().getFrameHeight(); + inImgW_ = inputLayers_[0]->getOutput().getFrameWidth(); + + const BilinearInterpConfig& conf = config_.inputs(0).bilinear_interp_conf(); + if (inImgH_ == 0) { + inImgH_ = conf.img_size_y(); + } + if (inImgW_ == 0) { + inImgW_ = conf.img_size_x(); + } + + outImgH_ = conf.out_size_y(); + outImgW_ = conf.out_size_x(); + numChannels_ = conf.num_channels(); + + CHECK(outImgH_ > 0 && outImgW_ > 0); + CHECK(inImgH_ > 0 && inImgW_ > 0); + CHECK(numChannels_); + + ratioH_ = (outImgH_ > 1) ? + static_cast(inImgH_ - 1) / (outImgH_ - 1) : 0.f; + ratioW_ = (outImgW_ > 1) ? + static_cast(inImgW_ - 1) / (outImgW_ - 1) : 0.f; + + getOutput().setFrameHeight(outImgH_); + getOutput().setFrameWidth(outImgW_); + return outImgH_ * outImgW_ * numChannels_; +} + +bool BilinearInterpLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + CHECK_EQ(1, config_.inputs_size()); + + return true; +} + +void BilinearInterpLayer::forward(PassType passType) { + Layer::forward(passType); + + size_t batchSize = getInput(0).getBatchSize(); + size_t size = getSize(); + { + REGISTER_TIMER_INFO("FwResetTimer", getName().c_str()); + resetOutput(batchSize, size); + } + + MatrixPtr inV = getInputValue(0); + MatrixPtr outV = getOutputValue(); + { + REGISTER_TIMER_INFO("FwBilinearInterpTimer", getName().c_str()); + outV->bilinearForward(*inV, inImgH_, inImgW_, outImgH_, outImgW_, + numChannels_, ratioH_, ratioW_); + } +} + +void BilinearInterpLayer::backward(const UpdateCallback& callback) { + (void) callback; + + MatrixPtr inputG = getInputGrad(0); + MatrixPtr outG = getOutputGrad(); + { + REGISTER_TIMER_INFO("BwBilinearInterpTimer", getName().c_str()); + if (inputG) { + inputG->bilinearBackward(*outG, outImgH_, outImgW_, inImgH_, inImgW_, + numChannels_, ratioH_, ratioW_); + } + } +} +} // namespace paddle diff --git a/paddle/gserver/layers/BilinearInterpLayer.h b/paddle/gserver/layers/BilinearInterpLayer.h new file mode 100644 index 0000000000000..eba3c054fa8e7 --- /dev/null +++ b/paddle/gserver/layers/BilinearInterpLayer.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "paddle/math/Matrix.h" + +namespace paddle { + +/** + * @brief A layer for bilinear interpolation which is + * used on conv layer output. + * + * @note The config file api is bilinear_interp_layer. + */ +class BilinearInterpLayer : public Layer { +protected: + size_t outImgH_, outImgW_; + size_t inImgH_, inImgW_; + real ratioH_, ratioW_; + size_t numChannels_; + +public: + explicit BilinearInterpLayer(const LayerConfig& config) : Layer(config) {} + + virtual ~BilinearInterpLayer() {} + + size_t getSize(); + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + void forward(PassType passType); + void backward(const UpdateCallback& callback = nullptr); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/ConcatenateLayer.cpp b/paddle/gserver/layers/ConcatenateLayer.cpp index 52a7cb6f777c3..a986ec10b4a01 100644 --- a/paddle/gserver/layers/ConcatenateLayer.cpp +++ b/paddle/gserver/layers/ConcatenateLayer.cpp @@ -97,7 +97,8 @@ void ConcatenateLayer::backward(const UpdateCallback& callback) { */ class ConcatenateLayer2 : public Layer { public: - explicit ConcatenateLayer2(const LayerConfig& config) : Layer(config) {} + explicit ConcatenateLayer2(const LayerConfig& config) : + Layer(config) {} ~ConcatenateLayer2() {} @@ -110,6 +111,8 @@ class ConcatenateLayer2 : public Layer { std::vector> projections_; std::vector projOutput_; std::vector> projCol_; + bool sharedBias_; + std::unique_ptr biases_; }; REGISTER_LAYER(concat2, ConcatenateLayer2); @@ -119,7 +122,6 @@ bool ConcatenateLayer2::init(const LayerMap& layerMap, /* Initialize the basic parent class */ if (!Layer::init(layerMap, parameterMap)) return false; - CHECK(!biasParameter_); CHECK_EQ(inputLayers_.size(), parameters_.size()); projections_.reserve(inputLayers_.size()); projCol_.reserve(inputLayers_.size()); @@ -137,6 +139,13 @@ bool ConcatenateLayer2::init(const LayerMap& layerMap, } CHECK_EQ(getSize(), endCol); + /* initialize biases_ */ + if (biasParameter_.get() != NULL) { + sharedBias_ = config_.shared_biases(); + size_t psize = config_.bias_size(); + biases_ = std::unique_ptr(new Weight(1, psize, biasParameter_)); + } + return true; } @@ -151,11 +160,22 @@ void ConcatenateLayer2::forward(PassType passType) { size_t startCol = projCol_[i].first; size_t endCol = projCol_[i].second; projOutput_[i].value = output_.value->subColMatrix(startCol, endCol); - projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); + if (output_.grad) { + projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); + } } - for (size_t i = 0; i != inputLayers_.size(); ++i) { - projections_[i]->forward(&getInput(i), &projOutput_[i], passType); + { + AsyncGpuBlock block; + for (size_t i = 0; i != inputLayers_.size(); ++i) { + projections_[i]->forward(&getInput(i), &projOutput_[i], passType); + } + } + + /* add the bias-vector */ + if (biases_) { + REGISTER_TIMER_INFO("FwBiasTimer", getName().c_str()); + output_.value->addBias(*(biases_->getW()), 1, sharedBias_); } /* activation */ { @@ -170,6 +190,13 @@ void ConcatenateLayer2::backward(const UpdateCallback& callback) { backwardActivation(); } + AsyncGpuBlock block; + if (biases_ && biases_->getWGrad()) { + REGISTER_TIMER_INFO("Concat2BpBiasTimer", getName().c_str()); + biases_->getWGrad()->collectBias(*getOutputGrad(), 1, sharedBias_); + biases_->getParameterPtr()->incUpdate(callback); + } + for (size_t i = 0; i != inputLayers_.size(); ++i) { if (projections_[i]) { projections_[i]->backward(callback); diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 9ed9572139dc8..6bc3b3b801796 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -12,15 +12,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/utils/Logging.h" #include "ConvBaseLayer.h" +#include "paddle/math/MathUtils.h" namespace paddle { bool ConvBaseLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { /* Initialize the basic parent class */ Layer::init(layerMap, parameterMap); + isDeconv_ = (config_.type() == "exconv" || config_.type() == "cudnn_conv") + ? false : true; /* Initialize the convolutional layer parameter */ numFilters_ = config_.num_filters(); @@ -35,20 +37,19 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, filterSizeY_.push_back(conf.filter_size_y()); filterPixels_.push_back(filterSize_.back() * filterSizeY_.back()); channels_.push_back(conf.channels()); - imgSize_.push_back(conf.img_size()); - imgPixels_.push_back(imgSize_.back() * imgSize_.back()); + imgSizeH_.push_back(conf.img_size()); + imgSizeW_.push_back(conf.img_size()); groups_.push_back(conf.groups()); filterChannels_.push_back(conf.filter_channels()); - outputX_.push_back(conf.output_x()); - outputs_.push_back(outputX_.back() * outputX_.back()); + outputH_.push_back(conf.output_x()); + outputW_.push_back(conf.output_x()); } - /* initialize the weightList */ CHECK(inputLayers_.size() == parameters_.size()); for (size_t i = 0; i < inputLayers_.size(); i++) { size_t height, width; height = filterPixels_[i] * filterChannels_[i]; - width = numFilters_; + width = (!isDeconv_) ? numFilters_ : channels_[i]; // create a new weight CHECK_EQ(parameters_[i]->getSize(), width * height); @@ -57,7 +58,7 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, } /* initialize the biases_ */ - if (biasParameter_.get() != NULL) { + if (biasParameter_.get()) { if (sharedBiases_) { CHECK_EQ((size_t)numFilters_, biasParameter_->getSize()); biases_ = @@ -74,4 +75,59 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, return true; } +size_t ConvBaseLayer::calOutputSize() { + auto clearAndReserve = [this](IntV* vec) { + vec->clear(); + vec->reserve(this->inputLayers_.size()); + }; + clearAndReserve(&imgSizeH_); + clearAndReserve(&imgSizeW_); + clearAndReserve(&outputH_); + clearAndReserve(&outputW_); + size_t layerSize = 0; + + auto setLayerSize = [&](IntV& inH, IntV& inW, IntV& outH, IntV& outW) { + for (size_t i = 0; i < inputLayers_.size(); i++) { + inH.push_back(inputLayers_[i]->getOutput().getFrameHeight()); + inW.push_back(inputLayers_[i]->getOutput().getFrameWidth()); + if (isDeconv_) { + if (inH[i] == 0) + inH[i] = config_.inputs(i).conv_conf().output_x(); + if (inW[i] == 0) + inW[i] = config_.inputs(i).conv_conf().output_x(); + outH.push_back( + imageSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i], + caffeMode_)); + outW.push_back( + imageSize(inW[i], filterSize_[i], padding_[i], stride_[i], + caffeMode_)); + } else { + if (inH[i] == 0) + inH[i] = config_.inputs(i).conv_conf().img_size(); + if (inW[i] == 0) + inW[i] = config_.inputs(i).conv_conf().img_size(); + outH.push_back( + outputSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i], + caffeMode_)); + outW.push_back( + outputSize(inW[i], filterSize_[i], padding_[i], stride_[i], + caffeMode_)); + } + CHECK_EQ(outH[i], outH[0]); + CHECK_EQ(outW[i], outW[0]); + } + getOutput().setFrameHeight(outH[0]); + getOutput().setFrameWidth(outW[0]); + layerSize = outH[0] * outW[0] * size_t(numFilters_); + }; + + if (isDeconv_) { + setLayerSize(outputH_, outputW_, imgSizeH_, imgSizeW_); + } else { + setLayerSize(imgSizeH_, imgSizeW_, outputH_, outputW_); + } + + return layerSize; +} + } // namespace paddle diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index eaeaebf43be25..b80cab899585e 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -16,6 +16,7 @@ limitations under the License. */ #pragma once #include "Layer.h" +#include "paddle/math/MathUtils.h" namespace paddle { /** @@ -27,6 +28,9 @@ class ConvBaseLayer : public Layer { protected: typedef std::vector IntV; + /// True if it's deconv layer, false if it's convolution layer + bool isDeconv_; + /// The number of filters. int numFilters_; /// The x dimension of the padding. @@ -43,19 +47,18 @@ class ConvBaseLayer : public Layer { IntV filterSizeY_; /// The spatial dimensions of the convolution input. IntV channels_; - /// The spatial dimensions of input feature map. - IntV imgSize_; - /// The total pixel size of input feature map. - /// imgPixels_ = imgSizeX_ * imgSizeY_. - IntV imgPixels_; + /// The spatial dimensions of input feature map height. + IntV imgSizeH_; + /// The spatial dimensions of input feature map width. + IntV imgSizeW_; /// filterPixels_ = filterSizeX_ * filterSizeY_. IntV filterPixels_; /// filterChannels_ = channels_/groups_. IntV filterChannels_; - /// The spatial dimensions of output feature map. - IntV outputX_; - /// The spatial dimensions of output feature map. - IntV outputs_; + /// The spatial dimensions of output feature map height. + IntV outputH_; + /// The spatial dimensions of output feature map width. + IntV outputW_; /// Group size, refer to grouped convolution in /// Alex Krizhevsky's paper: when group=2, the first half of the /// filters are only connected to the first half of the input channels, @@ -80,32 +83,14 @@ class ConvBaseLayer : public Layer { virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - Weight& getWeight(int idx) { return *weights_[idx]; } - /** - * Calculate output size based on caffeMode_. - * - input(+padding): 0123456789 - * - imageSize(+padding) = 10; - * - filterSize = 3; - * - stride = 2; - * - caffeMode_ is true: - - output: (012), (234), (456), (678) - - outputSize = 4; - * - caffeMode_ is false: - * - output: (012), (234), (456), (678), (9) - * - outputSize = 5; + * imgSizeH_ and imgSizeW_ will be set according to the previous input layers + * in this function. Then it will calculate outputH_ and outputW_ and set them + * into output argument. */ - int outputSize(int imageSize, int filterSize, int padding, int stride) { - int outputSize; - if (!caffeMode_) { - outputSize = - (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; - } else { - outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; - } - CHECK_GE(outputSize, 1); - return outputSize; - } + virtual size_t calOutputSize(); + + Weight& getWeight(int idx) { return *weights_[idx]; } }; } // namespace paddle diff --git a/paddle/gserver/layers/ConvOperator.cpp b/paddle/gserver/layers/ConvOperator.cpp index 8c72c1778451d..2d9c892fe595f 100644 --- a/paddle/gserver/layers/ConvOperator.cpp +++ b/paddle/gserver/layers/ConvOperator.cpp @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/math/Matrix.h" +#include "paddle/math/MathUtils.h" #include "Operator.h" namespace paddle { @@ -35,8 +35,8 @@ class ConvOperator : public Operator { */ virtual ~ConvOperator() { if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpace_); - workSpaceInBytes_ = 0; + hl_free_mem_device(workSpace_); + workSpaceInBytes_ = 0; } hl_destroy_tensor_descriptor(inputDesc_); @@ -83,33 +83,6 @@ class ConvOperator : public Operator { filterSize_ * filterSizeY_ * channels_ * numFilters_); } - /** - * Calculate output size. - */ - int outputSize(int imageSize, int filterSize, int padding, int stride) { - int outputSize; - if (!caffeMode_) { - /* input(+padding): 0123456789 - * imageSize(+padding) = 10; - * filterSize = 3; - * stride = 2; - * output: (012), (234), (456), (678), (9) - * outputSize = 5; - */ - outputSize = - (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; - } else { - /* input(+padding): 0123456789 - * imageSize(+padding) = 10; - * filterSize = 3; - * stride = 2; - * output: (012), (234), (456), (678) - * outputSize = 4; - */ - outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; - } - return outputSize; - } /// Most of member variables are same with CudnnConvLayer. /// There is no explanation here. int imageH_, imageW_, outputH_, outputW_; @@ -129,7 +102,7 @@ class ConvOperator : public Operator { int fwdAlgo_, bwdFilterAlgo_, bwdDataAlgo_; size_t fwdLimitBytes_, bwdDataLimitBytes_, bwdFilterLimitBytes_; size_t workSpaceInBytes_; - void* workSpace_; + void *workSpace_; bool isSelectAlgo_; }; @@ -160,7 +133,7 @@ ConvOperator::ConvOperator(const OperatorConfig &config, bool useGpu) void ConvOperator::allocConvWorkSpace(size_t maxWorkSpace) { if (maxWorkSpace > workSpaceInBytes_) { if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpace_); + hl_free_mem_device(workSpace_); } // total amount of storage needed workSpace_ = hl_malloc_device(maxWorkSpace); @@ -168,14 +141,13 @@ void ConvOperator::allocConvWorkSpace(size_t maxWorkSpace) { } } - void ConvOperator::reshape(int batchSize) { imageH_ = ins_[0]->getFrameHeight(); imageW_ = ins_[0]->getFrameWidth(); if (imageH_ == 0) imageH_ = imgSize_; if (imageW_ == 0) imageW_ = imgSize_; - outputH_ = outputSize(imageH_, filterSizeY_, paddingY_, strideY_); - outputW_ = outputSize(imageW_, filterSize_, padding_, stride_); + outputH_ = outputSize(imageH_, filterSizeY_, paddingY_, strideY_, caffeMode_); + outputW_ = outputSize(imageW_, filterSize_, padding_, stride_, caffeMode_); out_->setFrameHeight(outputH_); out_->setFrameWidth(outputW_); @@ -183,10 +155,10 @@ void ConvOperator::reshape(int batchSize) { reshapeImageDescriptors(); if (!isSelectAlgo_) { - hl_conv_workspace(inputDesc_, outputDesc_, filterDesc_, - convDesc_, &fwdAlgo_, &fwdLimitBytes_, - &bwdDataAlgo_, &bwdDataLimitBytes_, - &bwdFilterAlgo_, &bwdFilterLimitBytes_); + hl_conv_workspace(inputDesc_, outputDesc_, filterDesc_, convDesc_, + &fwdAlgo_, &fwdLimitBytes_, &bwdDataAlgo_, + &bwdDataLimitBytes_, &bwdFilterAlgo_, + &bwdFilterLimitBytes_); size_t maxWorkSpace = 0; maxWorkSpace = std::max(fwdLimitBytes_, bwdDataLimitBytes_); @@ -202,7 +174,8 @@ void ConvOperator::computeConvSizes() { hl_create_filter_descriptor(&filterDesc_, channels_, numFilters_, filterSizeY_, filterSize_); hl_create_tensor_descriptor(&inputDesc_); - int outputX = outputSize(imgSize_, filterSize_, padding_, stride_); + int outputX = + outputSize(imgSize_, filterSize_, padding_, stride_, caffeMode_); CHECK_EQ(outputX, outputX_); hl_create_tensor_descriptor(&outputDesc_); hl_create_convolution_descriptor(&convDesc_, inputDesc_, filterDesc_, @@ -211,13 +184,13 @@ void ConvOperator::computeConvSizes() { void ConvOperator::reshapeImageDescriptors() { hl_tensor_reshape(inputDesc_, 1, channels_, imageH_, imageW_, - channels_ * imageH_ * imageW_, imageH_ * imageW_, - imageW_, 1); + channels_ * imageH_ * imageW_, imageH_ * imageW_, imageW_, + 1); hl_tensor_reshape(outputDesc_, 1, numFilters_, outputH_, outputW_, numFilters_ * outputH_ * outputW_, outputH_ * outputW_, outputW_, 1); - hl_reset_convolution_descriptor(convDesc_, inputDesc_, filterDesc_, - paddingY_, padding_, strideY_, stride_); + hl_reset_convolution_descriptor(convDesc_, inputDesc_, filterDesc_, paddingY_, + padding_, strideY_, stride_); inputOffset_ = channels_ * imageH_ * imageW_; outputOffset_ = numFilters_ * outputH_ * outputW_; weightOffset_ = numFilters_ * channels_ * filterSize_ * filterSize_; @@ -273,18 +246,17 @@ void ConvOperator::backward() { real *weightGrad = ins_[1]->grad->getData() + weightOffset_ * batchId; hl_convolution_backward_filter(inputDesc_, inputData, outputDesc_, outGrad, filterDesc_, weightGrad, - convDesc_, workSpace_, - workSpaceInBytes_, bwdFilterAlgo_); + convDesc_, workSpace_, workSpaceInBytes_, + bwdFilterAlgo_); } MatrixPtr preGrad = ins_[0]->grad; if (NULL != preGrad) { real *inputGrad = preGrad->getData() + inputOffset_ * batchId; real *wgtData = ins_[1]->value->getData() + weightOffset_ * batchId; - hl_convolution_backward_data(inputDesc_, inputGrad, outputDesc_, - outGrad, filterDesc_, wgtData, - convDesc_, workSpace_, - workSpaceInBytes_, bwdDataAlgo_); + hl_convolution_backward_data( + inputDesc_, inputGrad, outputDesc_, outGrad, filterDesc_, wgtData, + convDesc_, workSpace_, workSpaceInBytes_, bwdDataAlgo_); } } } diff --git a/paddle/gserver/layers/ConvProjection.cpp b/paddle/gserver/layers/ConvProjection.cpp new file mode 100644 index 0000000000000..d1ce53fe26351 --- /dev/null +++ b/paddle/gserver/layers/ConvProjection.cpp @@ -0,0 +1,210 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "paddle/utils/Stat.h" +#include "ConvProjection.h" + +namespace paddle { + +REGISTER_PROJECTION(conv, ConvProjection); + +ThreadLocalD> ConvProjection::convMem_; + +ConvProjection::ConvProjection(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) + : Projection(config, parameter, useGpu) { + + CHECK(useGpu); // only support GPU + getConvParams(); + initCudnn(); + + size_t height = filterH_ * filterW_ * channels_ / groups_; + size_t width = numFilters_; + weight_.reset(new Weight(height, width, parameter)); + weightOffset_ = height * width / groups_; +} + +void ConvProjection::getConvParams() { + const ConvConfig &conf = config_.conv_conf(); + paddingH_ = conf.padding_y(); + paddingW_ = conf.padding(); + + strideH_ = conf.stride_y(); + strideW_ = conf.stride(); + + filterH_ = conf.filter_size_y(); + filterW_ = conf.filter_size(); + + configImgH_ = conf.img_size(); + configImgW_ = conf.img_size(); + + channels_ = conf.channels(); + numFilters_ = config_.num_filters(); + + groups_ = conf.groups(); + CHECK_EQ(channels_ % groups_, 0); + CHECK_EQ(numFilters_ % groups_, 0); +} + +void ConvProjection::initCudnn() { + hl_create_filter_descriptor(&filterDesc_, channels_, numFilters_, + filterH_, filterW_); + hl_create_tensor_descriptor(&inputDesc_); + hl_create_tensor_descriptor(&outputDesc_); + hl_create_convolution_descriptor(&convDesc_, inputDesc_, filterDesc_, + paddingH_, paddingW_, strideH_, strideW_); + + // initialize all to default algorithms + fwdAlgo_ = 0; + bwdFilterAlgo_ = 0; + bwdDataAlgo_ = 0; + fwdLimitBytes_ = 0; + bwdDataLimitBytes_ = 0; + bwdFilterLimitBytes_ = 0; + workSpaceInBytes_ = 0; + + batchNum_ = 0; + isSelectAlgo_ = false; +} + +void ConvProjection::reshapeTensorDesc(int batchSize) { + hl_tensor_reshape(inputDesc_, batchSize, channels_, imageH_, imageW_, + channels_ * imageH_ * imageW_, imageH_ * imageW_, + imageW_, 1); + hl_reset_convolution_descriptor(convDesc_, inputDesc_, filterDesc_, + paddingH_, paddingW_, strideH_, strideW_); + + // The stride between two consecutive images in ConvProjection may not be 1, + // for example, in the case of layer ConcatenateLayer2 with two + // ConvProjection, the stride is the output_size of layer ConcatenateLayer2. + // So the calculation of nStride is different from CudnnConvLayer. + // In fact, only "nStride = out_->value->getStride()" is ok. + size_t nStride = numFilters_ * outputH_ * outputW_; + if (out_->value->isContiguous()) { + CHECK_EQ(nStride, out_->value->getWidth()); + } else { + nStride = out_->value->getStride(); + } + + hl_tensor_reshape(outputDesc_, batchSize, numFilters_, outputH_, outputW_, + nStride, outputH_ * outputW_, outputW_, 1); +} + +void ConvProjection::reshape(int batchSize) { + size_t width = calOutputSize(); + CHECK_EQ(width, out_->value->getWidth()); + + isSelectAlgo_ = (batchSize == batchNum_); + batchNum_ = batchSize; + + if (!isSelectAlgo_) { + reshapeTensorDesc(batchSize); + hl_conv_workspace(inputDesc_, outputDesc_, filterDesc_, + convDesc_, &fwdAlgo_, &fwdLimitBytes_, + &bwdDataAlgo_, &bwdDataLimitBytes_, + &bwdFilterAlgo_, &bwdFilterLimitBytes_); + + size_t maxWorkSpace = 0; + maxWorkSpace = std::max(fwdLimitBytes_, bwdDataLimitBytes_); + maxWorkSpace = std::max(maxWorkSpace, bwdFilterLimitBytes_); + workSpaceInBytes_ = maxWorkSpace; + + + VLOG(3) << getName() << " Fwd / BwdData / BwdFilter algo: " << fwdAlgo_ + << " / " << bwdDataAlgo_ + << " / " << bwdFilterAlgo_; + } + + isSelectAlgo_ = true; +} + +void ConvProjection::forward() { + int batchSize = in_->value->getHeight(); + reshape(batchSize); + + void* workSpace = NULL; + if (workSpaceInBytes_ > 0) { + workSpace = getSpaceBytes(workSpaceInBytes_); + } + + for (int g = 0; g < groups_; ++g) { + REGISTER_TIMER_INFO("CudnnConvFwTimer", getName().c_str()); + + real *inputData = in_->value->getData() + g * inputOffset_; + real *wgtData = weight_->getW()->getData() + g * weightOffset_; + real *outData = out_->value->getData() + g * outputOffset_; + hl_convolution_forward(inputDesc_, inputData, outputDesc_, + outData, filterDesc_, wgtData, + convDesc_, workSpace, + fwdLimitBytes_, fwdAlgo_); + } +} + +void ConvProjection::backward(const UpdateCallback& callback) { + REGISTER_TIMER_INFO("CudnnConvBpTimer", getName().c_str()); + + void* workSpace = NULL; + if (workSpaceInBytes_ > 0) { + workSpace = getSpaceBytes(workSpaceInBytes_); + } + + for (int g = 0; g < groups_; ++g) { + real *outGrad = out_->grad->getData() + g * outputOffset_; + if (weight_->getWGrad()) { + real *inputData = in_->value->getData() + g * inputOffset_; + real *weightGrad = weight_->getWGrad()->getData() + g * weightOffset_; + hl_convolution_backward_filter( + inputDesc_, inputData, outputDesc_, outGrad, filterDesc_, + weightGrad, convDesc_, workSpace, bwdFilterLimitBytes_, + bwdFilterAlgo_); + } + + MatrixPtr preGrad = in_->grad; + if (NULL != preGrad) { + real *inputGrad = preGrad->getData() + g * inputOffset_; + real *wgtData = weight_->getW()->getData() + g* weightOffset_; + hl_convolution_backward_data( + inputDesc_, inputGrad, outputDesc_, outGrad, filterDesc_, + wgtData, convDesc_, workSpace, bwdDataLimitBytes_, + bwdDataAlgo_); + } + } + + weight_->getParameterPtr()->incUpdate(callback); +} + +void* ConvProjection::getSpaceBytes(size_t size) { + std::vector& convMem = *convMem_; + if (convMem.empty()) { + int numDevices = hl_get_device_count(); + convMem.resize(numDevices); + } + + int devId = hl_get_device(); + MemoryHandle** localMem = &(convMem[devId]); + if (NULL == *localMem || size > (*localMem)->getAllocSize()) { + *localMem = new GpuMemoryHandle(size); + } + return (*localMem)->getBuf(); +} + +ConvProjection::~ConvProjection() { + hl_destroy_tensor_descriptor(inputDesc_); + hl_destroy_tensor_descriptor(outputDesc_); + hl_destroy_filter_descriptor(filterDesc_); + hl_destroy_convolution_descriptor(convDesc_); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ConvProjection.h b/paddle/gserver/layers/ConvProjection.h new file mode 100644 index 0000000000000..d0bfe9a6edba0 --- /dev/null +++ b/paddle/gserver/layers/ConvProjection.h @@ -0,0 +1,123 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Projection.h" +#include "paddle/math/MathUtils.h" + +namespace paddle { + +/** + * @brief Convolution projection do the same calculation with CudnnConvLayer. + */ +class ConvProjection : public Projection { +public: + /** + * Constructor. + */ + ConvProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu); + + ~ConvProjection(); + + virtual void forward(); + virtual void backward(const UpdateCallback& callback); + +protected: + void getConvParams(); + void initCudnn(); + + void reshapeTensorDesc(int batchSize); + void reshape(int batchSize); + + size_t calOutputSize() { + imageH_ = in_->getFrameHeight(); + imageW_ = in_->getFrameWidth(); + if (imageH_ == 0) imageH_ = configImgH_; + if (imageW_ == 0) imageW_ = configImgW_; + outputH_ = outputSize(imageH_, filterH_, paddingH_, strideH_, + /* caffeMode */ true); + outputW_ = outputSize(imageW_, filterW_, paddingW_, strideW_, + /* caffeMode */ true); + + const_cast(out_)->setFrameHeight(outputH_); + const_cast(out_)->setFrameWidth(outputW_); + + inputOffset_ = (channels_ / groups_) * imageH_ * imageW_; + outputOffset_ = (numFilters_ / groups_) * outputH_ * outputW_; + return outputH_ * outputW_ * numFilters_; + } + + static void* getSpaceBytes(size_t size); + + /// imageH_ and imageW_ is calculated from the input layer. + int imageH_, imageW_; + /// configImgH_ and configImgW_ is obtained from config. + int configImgH_, configImgW_; + int outputH_, outputW_; + int channels_, numFilters_; + int paddingH_, paddingW_; + int strideH_, strideW_; + int filterH_, filterW_; + /// One group offset of input data. + int inputOffset_; + /// One group offset of output data. + int outputOffset_; + /// One group offset of weight. + int weightOffset_; + int groups_; + + /// Cudnn tensor descriptor for input. + hl_tensor_descriptor inputDesc_; + /// Cudnn tensor descriptor for output. + hl_tensor_descriptor outputDesc_; + /// Cudnn tensor descriptor for filter. + hl_filter_descriptor filterDesc_; + /// Cudnn tensor descriptor for a convolution operation. + hl_convolution_descriptor convDesc_; + + /// Record the algorithm for forward convolution, which is obtained by cudnn + /// api to search the best suited algorithm. + int fwdAlgo_; + /// Record the algorithm for computing convolution gradient with respect to + /// filter coefficients. + int bwdFilterAlgo_; + /// Record the algorithm for computing convolution gradient with respect to + /// the output. + int bwdDataAlgo_; + /// Amount of GPU memory needed as workspace to be able to execute a + /// forward convolution with the specified algo. + size_t fwdLimitBytes_; + /// Amount of GPU memory needed as workspace to be able to execute a + /// backwardFilter with the specified algo. + size_t bwdDataLimitBytes_; + /// Amount of GPU memory needed as workspace to be able to execute a + /// backwardData with the specified algo. + size_t bwdFilterLimitBytes_; + /// Size of total work space. + size_t workSpaceInBytes_; + + /// Whether to call cuDNN api to choose conv algorithm. + bool isSelectAlgo_; + /// batchNum is used to record batch size. If the batch size is changed, + /// the selection algorithm will be called. + int batchNum_; + bool bias_; + + std::unique_ptr weight_; + static ThreadLocalD> convMem_; +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 14ff8510f7b19..3c2df52fed4f8 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -462,25 +462,43 @@ bool MultiBinaryLabelCrossEntropy::init(const LayerMap& layerMap, void MultiBinaryLabelCrossEntropy::forwardImp(Matrix& output, Argument& label, Matrix& target) { - if (dynamic_cast(label.value.get()) || - dynamic_cast(label.value.get())) { - target.multiBinaryLabelCrossEntropy(output, *label.value); + MatrixPtr value = nullptr; + if (label.ids) { + CHECK(!label.value); + value = label.ids->toOneHotSparseMatrix(output.getWidth(), useGpu_); + } else { + CHECK(label.value); + value = label.value; + } + + if (dynamic_cast(value.get()) || + dynamic_cast(value.get())) { + target.multiBinaryLabelCrossEntropy(output, *value); } else { Matrix::resizeOrCreate(targetPerDim_, output.getHeight(), output.getWidth(), false, useGpu_); - targetPerDim_->binaryLabelCrossEntropy(output, *label.value); + targetPerDim_->binaryLabelCrossEntropy(output, *value); targetPerDim_->rowSum(target); } } void MultiBinaryLabelCrossEntropy::backwardImp( Matrix& output, Argument& label, Matrix& outputG) { - if (dynamic_cast(label.value.get()) || - dynamic_cast(label.value.get())) { - outputG.multiBinaryLabelCrossEntropyBp(output, *label.value); + MatrixPtr value = nullptr; + if (label.ids) { + CHECK(!value); + value = label.ids->toOneHotSparseMatrix(output.getWidth(), useGpu_); } else { - outputG.binaryLabelCrossEntropyBp(output, *label.value); + CHECK(label.value); + value = label.value; + } + + if (dynamic_cast(value.get()) || + dynamic_cast(value.get())) { + outputG.multiBinaryLabelCrossEntropyBp(output, *value); + } else { + outputG.binaryLabelCrossEntropyBp(output, *value); } } @@ -562,4 +580,39 @@ void HuberTwoClass::backwardImpIn( } } +/** + * This cost layer compute the sum of its input as loss. + * \f[ + * o(i) = \sum_{j=1}^D y_{ij} + * \f] + */ +class SumCostLayer : public Layer { +public: + explicit SumCostLayer(const LayerConfig& config) : Layer(config) {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap) { + bool ret = Layer::init(layerMap, parameterMap); + if (!ret) return ret; + CHECK_EQ(inputLayers_.size(), 1UL); + return true; + } + + virtual void forward(PassType passType) { + Layer::forward(passType); + const MatrixPtr& input = getInputValue(0); + + /* malloc memory for the output_ if necessary */ + int batchSize = input->getHeight(); + int size = 1; + resizeOutput(batchSize, size); + output_.value->sumRows(*input, /* scaleSum= */1, /* scaleDest= */0); + } + + virtual void backward(const UpdateCallback& callback = nullptr) { + getInputGrad(0)->add((real)1); + } +}; + +REGISTER_LAYER(sum_cost, SumCostLayer); + } // namespace paddle diff --git a/paddle/gserver/layers/CostLayer.h b/paddle/gserver/layers/CostLayer.h index b464e16737ae5..f263c688213ae 100644 --- a/paddle/gserver/layers/CostLayer.h +++ b/paddle/gserver/layers/CostLayer.h @@ -129,7 +129,7 @@ class SoftBinaryClassCrossEntropy : public CostLayer { * This cost layer compute Euclidean (L2) loss for real-valued regression * tasks. * \f[ - * L = \frac{1}{2N} \sum_{i=1}^N {|| \hat{y}_i - y_i||_2^2} + * L = \sum_{i=1}^N {|| \hat{y}_i - y_i||_2^2} * \f] */ class SumOfSquaresCostLayer : public CostLayer { diff --git a/paddle/gserver/layers/CudnnConvLayer.cpp b/paddle/gserver/layers/CudnnConvLayer.cpp index 0f932f960f6ba..23ba2341185d1 100644 --- a/paddle/gserver/layers/CudnnConvLayer.cpp +++ b/paddle/gserver/layers/CudnnConvLayer.cpp @@ -22,215 +22,64 @@ REGISTER_LAYER(cudnn_conv, CudnnConvLayer); bool CudnnConvLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { - ConvBaseLayer::init(layerMap, parameterMap); + if (!ConvBaseLayer::init(layerMap, parameterMap)) return false; CHECK(useGpu_) << "CudnnConvLayer only support gpu"; - maxGroups_ = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - CHECK_EQ(channels_[i] % groups_[i], 0); - CHECK_EQ(numFilters_ % groups_[i], 0); - - hl_filter_descriptor filter; - hl_create_filter_descriptor(&filter, channels_[i] / groups_[i], - numFilters_ / groups_[i], filterSizeY_[i], - filterSize_[i]); - filterDesc_.push_back(filter); - - hl_tensor_descriptor input; - hl_create_tensor_descriptor(&input); - inputDesc_.push_back(input); - - hl_tensor_descriptor output; - int outputX = - outputSize(imgSize_[i], filterSize_[i], padding_[i], stride_[i]); - CHECK_EQ(outputX, outputX_[i]); - hl_create_tensor_descriptor(&output); - outputDesc_.push_back(output); + CHECK_EQ(inputLayers_.size(), parameters_.size()); + projections_.reserve(inputLayers_.size()); + projConf_.reserve(inputLayers_.size()); - hl_convolution_descriptor conv; - hl_create_convolution_descriptor(&conv, input, filter, paddingY_[i], - padding_[i], strideY_[i], stride_[i]); - convDesc_.push_back(conv); - - weightOffset_.push_back((numFilters_ / groups_[i]) * - (channels_[i] / groups_[i]) * filterPixels_[i]); - inputOffset_.push_back((channels_[i] / groups_[i]) * imgSize_[i] * - imgSize_[i]); - outputOffset_.push_back((numFilters_ / groups_[i]) * outputX_[i] * - outputX_[i]); - - // initialize all to default algorithms - fwdAlgo_.push_back(0); - bwdFilterAlgo_.push_back(0); - bwdDataAlgo_.push_back(0); - fwdLimitBytes_.push_back(0); - bwdFilterLimitBytes_.push_back(0); - bwdDataLimitBytes_.push_back(0); - - // cudnn streams per group equal to 1 - if (groups_[i] > maxGroups_) { - maxGroups_ = groups_[i]; - } - } - - workSpaceInBytes_ = 0; - workSpaceData_ = NULL; - for (int i = 0; i < maxGroups_; ++i) { - workSpace_.push_back(NULL); + numFilters_ = config_.num_filters(); + CHECK(config_.shared_biases()); + for (size_t i = 0; i < inputLayers_.size(); i++) { + ProjectionConfig* conf = new ProjectionConfig(); + conf->set_type("conv"); + conf->set_num_filters(numFilters_); + ConvConfig* convConf = conf->mutable_conv_conf(); + *convConf = *(config_.mutable_inputs(i)->mutable_conv_conf()); + conf->set_input_size(getPrev(i)->getSize()); + conf->set_output_size(getSize()); + projConf_.emplace_back(conf); + projections_.emplace_back(Projection::create(*projConf_[i], + parameters_[i], useGpu_)); } if (biases_.get() && sharedBiases_) { hl_create_tensor_descriptor(&biasDesc_); + hl_create_tensor_descriptor(&outputDesc_); hl_tensor_reshape(biasDesc_, 1, numFilters_ / groups_[0], 1, 1); biasOffset_ = numFilters_ / groups_[0]; } - batchNum_ = 0; - isSelectAlgo_ = false; return true; } -void CudnnConvLayer::allocConvWorkSpace(size_t maxWorkSpace) { - size_t totalWorkSpace = maxWorkSpace * maxGroups_; - - if (totalWorkSpace > workSpaceInBytes_) { - if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpaceData_); - } - // total amount of storage needed over all groups - workSpaceData_ = hl_malloc_device(totalWorkSpace); - - // update work space address for each group - for (int i = 0; i < maxGroups_; ++i) { - workSpace_[i] = reinterpret_cast(workSpaceData_) - + i * maxWorkSpace; - } - workSpaceInBytes_ = totalWorkSpace; - } -} - -void CudnnConvLayer::reshape(int batchSize) { - CHECK_NE(inputLayers_.size(), 0UL); - imageH_ = inputLayers_[0]->getOutput().getFrameHeight(); - imageW_ = inputLayers_[0]->getOutput().getFrameWidth(); - if (imageH_ == 0) imageH_ = imgSize_[0]; - if (imageW_ == 0) imageW_ = imgSize_[0]; - - for (size_t i = 1; i < inputLayers_.size(); i++) { - int imageH = inputLayers_[i]->getOutput().getFrameHeight(); - int imageW = inputLayers_[i]->getOutput().getFrameWidth(); - if (imageH) { - CHECK_EQ(imageH_, imageH) << "Inputs must have same height."; - } - if (imageW) { - CHECK_EQ(imageW_, imageW) << "Inputs must have same width."; - } - } - - outputH_ = outputSize(imageH_, filterSizeY_[0], paddingY_[0], strideY_[0]); - outputW_ = outputSize(imageW_, filterSize_[0], padding_[0], stride_[0]); - // check outputH & outputW - getOutput().setFrameHeight(outputH_); - getOutput().setFrameWidth(outputW_); - - // if the batchSize remains the same, set isSelectAlgo_ true. - // Otherwise, set isSelectAlgo_ false and select algo again. - isSelectAlgo_ = (batchSize == batchNum_); - batchNum_ = batchSize; - - size_t maxWorkSpace = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - CHECK_EQ(inputLayers_[i]->getOutput().value->getWidth(), - (size_t)(channels_[i] * imageH_ * imageW_)); - - hl_tensor_reshape(inputDesc_[i], batchSize, channels_[i] / groups_[i], - imageH_, imageW_, channels_[i] * imageH_ * imageW_, - imageH_ * imageW_, imageW_, 1); - - hl_tensor_reshape(outputDesc_[i], batchSize, numFilters_ / groups_[i], - outputH_, outputW_, numFilters_ * outputH_ * outputW_, - outputH_ * outputW_, outputW_, 1); - - hl_reset_convolution_descriptor(convDesc_[i], inputDesc_[i], - filterDesc_[i], paddingY_[i], - padding_[i], strideY_[i], stride_[i]); - - inputOffset_[i] = (channels_[i] / groups_[i]) * imageH_ * imageW_; - outputOffset_[i] = (numFilters_ / groups_[i]) * outputH_ * outputW_; - - if (!isSelectAlgo_) { - hl_conv_workspace(inputDesc_[i], outputDesc_[i], filterDesc_[i], - convDesc_[i], &fwdAlgo_[i], &fwdLimitBytes_[i], - &bwdDataAlgo_[i], &bwdDataLimitBytes_[i], - &bwdFilterAlgo_[i], &bwdFilterLimitBytes_[i]); - - maxWorkSpace = std::max(fwdLimitBytes_[i], bwdDataLimitBytes_[i]); - maxWorkSpace = std::max(maxWorkSpace, bwdFilterLimitBytes_[i]); - - VLOG(3) << getName() << " Fwd / BwdData / BwdFilter algo: " << fwdAlgo_[i] - << " / " << bwdDataAlgo_[i] - << " / " << bwdFilterAlgo_[i]; - } - } - - if (!isSelectAlgo_) { - allocConvWorkSpace(maxWorkSpace); - } - - isSelectAlgo_ = true; -} - void CudnnConvLayer::forward(PassType passType) { Layer::forward(passType); - int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); - reshape(batchSize); - resetOutput(batchSize, outputH_ * outputW_ * numFilters_); + + int batchSize = getInput(0).getBatchSize(); + resetOutput(batchSize, calOutputSize()); for (size_t i = 0; i != inputLayers_.size(); ++i) { - REGISTER_TIMER_INFO("CudnnConvFwTimer", getName().c_str()); - for (int g = 0; g < groups_[i]; ++g) { - real *inputData = getInputValue(i)->getData() + inputOffset_[i] * g; - real *wgtData = weights_[i]->getW()->getData() + weightOffset_[i] * g; - real *outData = getOutputValue()->getData() + outputOffset_[i] * g; - hl_convolution_forward(inputDesc_[i], inputData, outputDesc_[i], - outData, filterDesc_[i], wgtData, - convDesc_[i], workSpace_[g], - fwdLimitBytes_[i], fwdAlgo_[i]); - } + projections_[i]->forward(&getInput(i), &getOutput(), passType); } if (biases_) { REGISTER_TIMER_INFO("CudnnConvBiasTimer", getName().c_str()); - addBiases(); - } - - forwardActivation(); -} - -void CudnnConvLayer::addBiases() { - if (sharedBiases_) { + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + hl_tensor_reshape(outputDesc_, batchSize, numFilters_ / groups_[0], + outputH_[0], outputW_[0], numFilters_ * outputH_[0] * outputW_[0], + outputH_[0] * outputW_[0], outputW_[0], 1); + outputOffset_ = getOutputValue()->getWidth() / groups_[0]; for (int g = 0; g < groups_[0]; ++g) { real *biasData = biases_->getW()->getData() + biasOffset_ * g; - real *outData = getOutputValue()->getData() + outputOffset_[0] * g; + real *outData = getOutputValue()->getData() + outputOffset_ * g; hl_convolution_forward_add_bias(biasDesc_, biasData, - outputDesc_[0], outData); + outputDesc_, outData); } - } else { - LOG(FATAL) << "Not supported"; } -} -void CudnnConvLayer::bpropBiases() { - if (sharedBiases_) { - for (int g = 0; g < groups_[0]; ++g) { - real *biasGrad = biases_->getWGrad()->getData() + biasOffset_ * g; - real *outGrad = getOutputGrad()->getData() + outputOffset_[0] * g; - hl_convolution_backward_bias(biasDesc_, biasGrad, - outputDesc_[0], outGrad); - } - } else { - LOG(FATAL) << "Not supported"; - } + forwardActivation(); } void CudnnConvLayer::backward(const UpdateCallback &callback) { @@ -238,52 +87,23 @@ void CudnnConvLayer::backward(const UpdateCallback &callback) { if (biases_ && biases_->getWGrad()) { REGISTER_TIMER_INFO("CudnnConvBpBiasTimer", getName().c_str()); - bpropBiases(); + for (int g = 0; g < groups_[0]; ++g) { + real *biasGrad = biases_->getWGrad()->getData() + biasOffset_ * g; + real *outGrad = getOutputGrad()->getData() + outputOffset_ * g; + hl_convolution_backward_bias(biasDesc_, biasGrad, outputDesc_, outGrad); + } biases_->getParameterPtr()->incUpdate(callback); } for (size_t i = 0; i != inputLayers_.size(); ++i) { - REGISTER_TIMER_INFO("CudnnConvBpTimer", getName().c_str()); - for (int g = 0; g < groups_[i]; ++g) { - real *outGrad = getOutputGrad()->getData() + outputOffset_[i] * g; - if (weights_[i]->getWGrad()) { - real *inputData = getInputValue(i)->getData() + inputOffset_[i] * g; - real *weightGrad = - weights_[i]->getWGrad()->getData() + weightOffset_[i] * g; - hl_convolution_backward_filter( - inputDesc_[i], inputData, outputDesc_[i], outGrad, filterDesc_[i], - weightGrad, convDesc_[i], workSpace_[g], bwdFilterLimitBytes_[i], - bwdFilterAlgo_[i]); - } - - MatrixPtr preGrad = getInputGrad(i); - if (NULL != preGrad) { - real *inputGrad = preGrad->getData() + inputOffset_[i] * g; - real *wgtData = weights_[i]->getW()->getData() + weightOffset_[i] * g; - hl_convolution_backward_data( - inputDesc_[i], inputGrad, outputDesc_[i], outGrad, filterDesc_[i], - wgtData, convDesc_[i], workSpace_[g], bwdDataLimitBytes_[i], - bwdDataAlgo_[i]); - } - } - weights_[i]->getParameterPtr()->incUpdate(callback); + projections_[i]->backward(callback); } } CudnnConvLayer::~CudnnConvLayer() { - if (biasDesc_) { + if (biases_) { hl_destroy_tensor_descriptor(biasDesc_); - } - - for (size_t i = 0; i < inputDesc_.size(); i++) { - hl_destroy_tensor_descriptor(inputDesc_[i]); - hl_destroy_tensor_descriptor(outputDesc_[i]); - hl_destroy_filter_descriptor(filterDesc_[i]); - hl_destroy_convolution_descriptor(convDesc_[i]); - } - if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpaceData_); - workSpaceInBytes_ = 0; + hl_destroy_tensor_descriptor(outputDesc_); } } diff --git a/paddle/gserver/layers/CudnnConvLayer.h b/paddle/gserver/layers/CudnnConvLayer.h index a6dadba10daa4..6390d96315cc4 100644 --- a/paddle/gserver/layers/CudnnConvLayer.h +++ b/paddle/gserver/layers/CudnnConvLayer.h @@ -17,12 +17,13 @@ limitations under the License. */ #include "ConvBaseLayer.h" #include "paddle/math/Matrix.h" +#include "Projection.h" #include namespace paddle { /** - * @brief A subclass of ConvBaseLayer by cuDNN implementation. It only + * @brief A 2-dimension conv layer implemented by cuDNN. It only * supports GPU mode. We automatic select CudnnConvLayer for GPU * mode and ExpandConvLayer for CPU mode if you set type of "conv". * User also can specfiy type of "exconv" or "cudnn_conv" for @@ -31,81 +32,21 @@ namespace paddle { * The config file api is img_conv_layer. */ class CudnnConvLayer : public ConvBaseLayer { -private: - /// resize Cudnn workspace size - void allocConvWorkSpace(size_t maxWorkSpace); - protected: - int imageH_, imageW_, outputH_, outputW_; - /// Cudnn tensor descriptor for bias. + std::vector> projConf_; + std::vector> projections_; + hl_tensor_descriptor biasDesc_; - /// Cudnn tensor descriptor for input. - std::vector inputDesc_; - /// Cudnn tensor descriptor for output. - std::vector outputDesc_; - /// Cudnn tensor descriptor for filter. - std::vector filterDesc_; - /// Cudnn tensor descriptor for a convolution operation. - std::vector convDesc_; - /// One sample offset of input data. - IntV inputOffset_; - /// One sample offset of output data. - IntV outputOffset_; - /// One group offset of weight. - IntV weightOffset_; - /// One group offset of bias. + hl_tensor_descriptor outputDesc_; int biasOffset_; - - /// Save the algorithm for forward convolution, which is obtained by cudnn - /// api to search the best suited algorithm. - std::vector fwdAlgo_; - /// Save the algorithm for computing convolution gradient with respect to - /// filter coefficients. - std::vector bwdFilterAlgo_; - /// Save the algorithm for computing convolution gradient with respect to - /// the output. - std::vector bwdDataAlgo_; - /// Amount of GPU memory needed as workspace to be able to execute a - /// forward convolution with the specified algo. - std::vector fwdLimitBytes_; - /// Amount of GPU memory needed as workspace to be able to execute a - /// backwardFilter with the specified algo. - std::vector bwdFilterLimitBytes_; - /// Amount of GPU memory needed as workspace to be able to execute a - /// backwardData with the specified algo. - std::vector bwdDataLimitBytes_; - - /// Device work space address for each group. - std::vector workSpace_; - /// Max number of groups. - int maxGroups_; - /// Total work space address in device for all groups. - void* workSpaceData_; - /// Size of total work space. - size_t workSpaceInBytes_; - - /// Is or not select conv algorihtm. - bool isSelectAlgo_; - - /// batchNum is used to record batch size. If the batch size is changed, - /// the selection algorithm will be called. - int batchNum_; + int outputOffset_; public: explicit CudnnConvLayer(const LayerConfig& config) : ConvBaseLayer(config) {} ~CudnnConvLayer(); - /** - * Intialization. Initialize member variables and create tenor descriptor. - */ bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - /** - * Reshape is done each forward. Reshape tensor decriptor - * inputDesc_, outputDesc_, convDesc_. And search the faster algo - * or the fastest algo within a given memeory limit. - */ - void reshape(int batchSize); void forward(PassType passType); void backward(const UpdateCallback& callback); void addBiases(); diff --git a/paddle/gserver/layers/CudnnPoolLayer.cpp b/paddle/gserver/layers/CudnnPoolLayer.cpp index 86c056ef5692a..24adb50a985ff 100644 --- a/paddle/gserver/layers/CudnnPoolLayer.cpp +++ b/paddle/gserver/layers/CudnnPoolLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" #include "paddle/math/Matrix.h" @@ -51,7 +50,6 @@ bool CudnnPoolLayer::init(const LayerMap &layerMap, PoolLayer::init(layerMap, parameterMap); CHECK(useGpu_) << "CudnnPoolLayer only support gpu"; - CHECK_EQ(start_, 0) << poolType_ << " dose not support 'start'"; hl_create_tensor_descriptor(&inputDesc_); hl_create_tensor_descriptor(&outputDesc_); @@ -63,9 +61,9 @@ bool CudnnPoolLayer::init(const LayerMap &layerMap, strideHeight = strideY_; strideWidth = stride_; - hl_create_pooling_descriptor(&poolingDesc_, mode_, windowHeight, - windowWidth, heightPadding, widthPadding, - strideHeight, strideWidth); + hl_create_pooling_descriptor(&poolingDesc_, mode_, windowHeight, windowWidth, + heightPadding, widthPadding, strideHeight, + strideWidth); return true; } @@ -81,8 +79,10 @@ void CudnnPoolLayer::reshape(int batchSize) { } CHECK_EQ(inputLayers_[0]->getOutput().value->getWidth(), channels_ * imageH_ * imageW_); - outputH_ = outputSize(imageH_, sizeY_, confPaddingY_, strideY_); - outputW_ = outputSize(imageW_, sizeX_, confPadding_, stride_); + outputH_ = outputSize(imageH_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputW_ = + outputSize(imageW_, sizeX_, confPadding_, stride_, /* caffeMode */ false); getOutput().setFrameHeight(outputH_); getOutput().setFrameWidth(outputW_); @@ -100,8 +100,7 @@ void CudnnPoolLayer::forward(PassType passType) { real *inputData = getInputValue(0)->getData(); real *outData = getOutputValue()->getData(); - hl_pooling_forward(inputDesc_, inputData, outputDesc_, outData, - poolingDesc_); + hl_pooling_forward(inputDesc_, inputData, outputDesc_, outData, poolingDesc_); } void CudnnPoolLayer::backward(const UpdateCallback &callback) { @@ -114,8 +113,8 @@ void CudnnPoolLayer::backward(const UpdateCallback &callback) { real *inputGrad = getInputGrad(0)->getData(); real *outData = getOutputValue()->getData(); real *outGrad = getOutputGrad()->getData(); - hl_pooling_backward(inputDesc_, inputData, inputGrad, outputDesc_, - outData, outGrad, poolingDesc_); + hl_pooling_backward(inputDesc_, inputData, inputGrad, outputDesc_, outData, + outGrad, poolingDesc_); } CudnnPoolLayer::~CudnnPoolLayer() { diff --git a/paddle/gserver/layers/CudnnPoolLayer.h b/paddle/gserver/layers/CudnnPoolLayer.h index df97ef2edfd01..2ef94720d2b9f 100644 --- a/paddle/gserver/layers/CudnnPoolLayer.h +++ b/paddle/gserver/layers/CudnnPoolLayer.h @@ -56,16 +56,6 @@ class CudnnPoolLayer : public PoolLayer { void reshape(int batchSize); virtual void forward(PassType passType); virtual void backward(const UpdateCallback& callback = nullptr); - - /** - * Calculate output size according window size of pooling. - */ - int outputSize(int imageSize, int windowSize, int padding, int stride) { - int outputSize; - outputSize = - (imageSize - windowSize + 2 * padding + stride - 1) / stride + 1; - return outputSize; - } }; } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvBaseLayer.cpp b/paddle/gserver/layers/ExpandConvBaseLayer.cpp new file mode 100644 index 0000000000000..0bab0ca764f4f --- /dev/null +++ b/paddle/gserver/layers/ExpandConvBaseLayer.cpp @@ -0,0 +1,263 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "ExpandConvBaseLayer.h" + +#include "paddle/utils/Logging.h" +namespace paddle { + +bool ExpandConvBaseLayer::init(const LayerMap &layerMap, + const ParameterMap ¶meterMap) { + /* Initialize the basic convolutional parent class */ + ConvBaseLayer::init(layerMap, parameterMap); + + /* The class fields channels_ and numFilters_ are the same as in the config + * i.e., channels_ is the for the input and numFilters_ is for the output + * + * But in order for the variables in convTrans having the same semantic + * meaning as in conv, we need to swap channels_ and numFilters here for + * convTrans, and in other functions too. + * */ + int channel; + int numFilters; + /* Initialize the projection */ + for (auto &inputConfig : config_.inputs()) { + const ConvConfig &conf = inputConfig.conv_conf(); + numFilters = isDeconv_ ? conf.channels() : numFilters_; + subM_.push_back(numFilters / conf.groups()); + subN_.push_back(conf.output_x() * conf.output_x()); + channel = isDeconv_ ? numFilters_ : conf.channels(); + subK_.push_back(channel * conf.filter_size() * conf.filter_size() / + conf.groups()); + /* Consistent caffe mode for multiple input */ + caffeMode_ = conf.caffe_mode(); + } + + getOutputSize(); + + return true; +} + +size_t ExpandConvBaseLayer::getOutputSize() { + CHECK_NE(inputLayers_.size(), 0UL); + size_t layerSize = ConvBaseLayer::calOutputSize(); + subN_.clear(); + for (size_t i = 0; i < inputLayers_.size(); i++) { + subN_.push_back(outputH_[i] * outputW_[i]); + } + return layerSize; +} + +void ExpandConvBaseLayer::resetExpandInput(size_t height, size_t width) { + Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); +} + +void ExpandConvBaseLayer::addSharedBias() { + size_t mapW = getOutputSize() / numFilters_; + size_t mapH = getOutputValue()->getElementCnt() / mapW; + MatrixPtr out = + Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + out->transpose(transOutValue_, false); // false means no memory allocation + transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, + numFilters_); + + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + transOutValue_->addBias(*bias, 1.0f); + + transOutValue_->reshape(mapW, mapH); + transOutValue_->transpose(out, false); // false means no memory allocation + + out->clear(); + bias->clear(); +} + +void ExpandConvBaseLayer::addUnsharedBias() { + MatrixPtr outValue = getOutputValue(); + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + outValue->addBias(*bias, 1.0f); +} + + +void ExpandConvBaseLayer::expandOneFrame(MatrixPtr image, size_t startIdx, + int inIdx) { + int channel = isDeconv_ ? numFilters_ : channels_[inIdx]; + + resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); + real *imgData = image->getData() + startIdx * image->getWidth(); + MatrixPtr imageTmp = Matrix::create( + imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channel, false, + useGpu_); + expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], + channel, filterSize_[inIdx], + filterSize_[inIdx], stride_[inIdx], stride_[inIdx], + padding_[inIdx], padding_[inIdx], + outputH_[inIdx], outputW_[inIdx]); + imageTmp->clear(); +} + +void ExpandConvBaseLayer::expandFwdOnce(MatrixPtr image, MatrixPtr out, + int inIdx, int startIdx) { + int subM = subM_[inIdx]; + int subN = subN_[inIdx]; + int subK = subK_[inIdx]; + + expandOneFrame(image, startIdx, inIdx); + + int numFilters = isDeconv_ ? channels_[inIdx] : numFilters_; + + real *outData = + out->getData() + startIdx * subN * numFilters; + + real *wgtData = weights_[inIdx]->getW()->getData(); + real *expInData = expandInput_->getData(); + for (int g = 0; g < groups_[inIdx]; ++g) { + MatrixPtr A = + Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose + MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); + MatrixPtr C = Matrix::create(outData, subM, subN, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + wgtData += subK * subM; + expInData += subK * subN; + outData += subM * subN; + } +} + +void ExpandConvBaseLayer::bpropActs(MatrixPtr out, MatrixPtr image, + int inpIdx) { + int channel = isDeconv_ ? numFilters_ : channels_[inpIdx]; + + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + size_t batchSize = image->getHeight(); + + /* reset the expand-grad memory */ + resetExpandInput(subK * groups_[inpIdx], subN); + + real *localGradData = out->getData(); + real *tgtGradData = image->getData(); + for (size_t n = 0; n < batchSize; n++) { + real *wgtData = weights_[inpIdx]->getW()->getData(); + real *expandInData = expandInput_->getData(); + + for (int g = 0; g < groups_[inpIdx]; g++) { + // create temporary matrix + MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(localGradData, subM, subN, false, useGpu_); + MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); + C->mul(A, B); // mul + + // clear the temporary matrix + A->clear(); + B->clear(); + C->clear(); + + expandInData += subK * subN; + localGradData += subM * subN; + wgtData += subK * subM; + } + + // shrink one frame outGrad + MatrixPtr oneGradTmp = Matrix::create( + expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); + MatrixPtr vTmp = Matrix::create( + tgtGradData, 1, + imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel, false, + useGpu_); + vTmp->convShrink(*oneGradTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], + channel, filterSize_[inpIdx], + filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], + padding_[inpIdx], padding_[inpIdx], + outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); + vTmp->clear(); + oneGradTmp->clear(); + + // move the data-pointer + tgtGradData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel; + } +} + +void ExpandConvBaseLayer::bpropWeights(MatrixPtr image, MatrixPtr out, + int inpIdx) { + MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); + + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + size_t batchSize = image->getHeight(); + resetExpandInput(subK * groups_[inpIdx], subN); + + real *gradData = out->getData(); + + for (size_t n = 0; n < batchSize; n++) { // frame by frame + // expand + expandOneFrame(image, n, inpIdx); + real *wGradData = weightGrad->getData(); + real *expandInData = expandInput_->getData(); + + // expand-mul one-group by one + for (int g = 0; g < groups_[inpIdx]; g++) { + MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(gradData, subM, subN, true, useGpu_); + MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + gradData += subM * subN; + wGradData += subK * subM; + expandInData += subK * subN; + } + } +} + +void ExpandConvBaseLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { + size_t mapW = getOutputSize() / numFilters_; + size_t mapH = v->getElementCnt() / mapW; + MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + vTmp->transpose(transOutValue_, false); // false means no memory allocation + transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, + numFilters_); + biases->collectBias(*transOutValue_, 1.0f); +} + +void ExpandConvBaseLayer::bpropBiases(MatrixPtr v) { + MatrixPtr biases = + Matrix::create(biases_->getWGrad()->getData(), 1, + biases_->getWGrad()->getElementCnt(), false, useGpu_); + if (sharedBiases_) { + bpropSharedBias(biases, v); + } else { + biases->collectBias(*v, 1.0f); + } + biases->clear(); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvBaseLayer.h b/paddle/gserver/layers/ExpandConvBaseLayer.h new file mode 100644 index 0000000000000..9858fa348c3fc --- /dev/null +++ b/paddle/gserver/layers/ExpandConvBaseLayer.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "ConvBaseLayer.h" +#include "paddle/math/Matrix.h" +#include + +namespace paddle { + +/** + * @brief A subclass of ConvBaseLayer that is a superclass of both + * ExpandConvLayer and ExpandConvTransLayer + */ +class ExpandConvBaseLayer : public ConvBaseLayer { +protected: + /// For expand convolution. + /// subM_ = numFilters_ / groups_. + IntV subM_; + /// subN_ = outputH_ * outputW_. + IntV subN_; + /// subK_ = channels_ * filterPixels_ * groups_. + IntV subK_; + + /*The expandInput_ and transOutValue_ are used for CPU expand conv calc + * Expand one sample at a time. shape: + * (numChannels * filterPixels_, outputSizeH * outputSizeW) + * */ + MatrixPtr expandInput_; + /// The transpose of output, which is an auxiliary matrix. + MatrixPtr transOutValue_; + +public: + explicit ExpandConvBaseLayer(const LayerConfig& config) + : ConvBaseLayer(config) {} + + ~ExpandConvBaseLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + size_t getOutputSize(); + /** + * Create or resize expandInput_. + */ + void resetExpandInput(size_t height, size_t width); + + /** + * Add shared bias. + */ + void addSharedBias(); + + /** + * Add unshared bias. + */ + void addUnsharedBias(); + /** + * Expand one input sample. + */ + void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); + + /** + * Expand one input sample and perform matrix multiplication. + */ + void expandFwdOnce(MatrixPtr image, MatrixPtr out, int inIdx, int startIdx); + + void bpropSharedBias(MatrixPtr biases, MatrixPtr v); + void bpropBiases(MatrixPtr v); + void bpropWeights(MatrixPtr image, MatrixPtr out, int inpIdx); + void bpropActs(MatrixPtr image, MatrixPtr out, int inpIdx); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index df79c3e3037cf..5ea1fdece5f7b 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -24,153 +24,29 @@ REGISTER_LAYER(exconv, ExpandConvLayer); bool ExpandConvLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ - ConvBaseLayer::init(layerMap, parameterMap); - - /* Initialize the projection */ - for (auto &inputConfig : config_.inputs()) { - const ConvConfig &conf = inputConfig.conv_conf(); - subM_.push_back(numFilters_ / conf.groups()); - subN_.push_back(conf.output_x() * conf.output_x()); - subK_.push_back(conf.channels() * conf.filter_size() * conf.filter_size() / - conf.groups()); - /* Consistent caffe mode for multiple input */ - caffeMode_ = conf.caffe_mode(); - } - + ExpandConvBaseLayer::init(layerMap, parameterMap); return true; } -size_t ExpandConvLayer::getSize() { - CHECK_NE(inputLayers_.size(), 0UL); - imgSizeH_.clear(); - imgSizeW_.clear(); - outputH_.clear(); - outputW_.clear(); - subN_.clear(); - size_t layerSize = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - imgSizeH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); - imgSizeW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); - if (imgSizeH_[i] == 0) imgSizeH_[i] = imgSize_[i]; - if (imgSizeW_[i] == 0) imgSizeW_[i] = imgSize_[i]; - outputH_.push_back( - outputSize(imgSizeH_[i], filterSize_[i], padding_[i], stride_[i])); - outputW_.push_back( - outputSize(imgSizeW_[i], filterSize_[i], padding_[i], stride_[i])); - subN_.push_back(outputH_[i] * outputW_[i]); - CHECK(layerSize == 0 || subN_[i] * size_t(numFilters_) == layerSize); - layerSize = subN_[i] * numFilters_; - } - getOutput().setFrameHeight(outputH_[0]); - getOutput().setFrameWidth(outputW_[0]); - return layerSize; -} - -void ExpandConvLayer::resetExpandInput(size_t height, size_t width) { - Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); -} - -void ExpandConvLayer::resetConvOutput(size_t batchSize, int inIdx) { - Matrix::resizeOrCreate(transOutValue_, batchSize * numFilters_, subN_[inIdx], - false, useGpu_); -} - -void ExpandConvLayer::expandOneFrame(MatrixPtr image, size_t startIdx, - int inIdx) { - resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); - real *imgData = image->getData() + startIdx * image->getWidth(); - MatrixPtr imageTmp = Matrix::create( - imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channels_[inIdx], false, - useGpu_); - expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], - channels_[inIdx], filterSize_[inIdx], - filterSize_[inIdx], stride_[inIdx], stride_[inIdx], - padding_[inIdx], padding_[inIdx], - outputH_[inIdx], outputW_[inIdx]); - imageTmp->clear(); -} - -void ExpandConvLayer::expandFwdOnce(MatrixPtr image, int inIdx, int startIdx) { - int subM = subM_[inIdx]; - int subN = subN_[inIdx]; - int subK = subK_[inIdx]; - - expandOneFrame(image, startIdx, inIdx); - - real *outData = - getOutputValue()->getData() + startIdx * subN * numFilters_; - - real *wgtData = weights_[inIdx]->getW()->getData(); - real *expInData = expandInput_->getData(); - for (int g = 0; g < groups_[inIdx]; ++g) { - MatrixPtr A = - Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose - MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); - MatrixPtr C = Matrix::create(outData, subM, subN, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - wgtData += subK * subM; - expInData += subK * subN; - outData += subM * subN; - } -} - -void ExpandConvLayer::addSharedBias() { - size_t mapW = getSize() / numFilters_; - size_t mapH = getOutputValue()->getElementCnt() / mapW; - MatrixPtr out = - Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - out->transpose(transOutValue_, false); // false means no memory allocation - transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, - numFilters_); - - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - transOutValue_->addBias(*bias, 1.0f); - - transOutValue_->reshape(mapW, mapH); - transOutValue_->transpose(out, false); // false means no memory allocation - - out->clear(); - bias->clear(); -} - -void ExpandConvLayer::addUnsharedBias() { - MatrixPtr outValue = getOutputValue(); - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - outValue->addBias(*bias, 1.0f); -} - void ExpandConvLayer::forward(PassType passType) { Layer::forward(passType); /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one colum, and the - * transOutValue correspond sample to one row */ - int batchSize = inputLayers_[0]->getOutputValue()->getWidth(); - batchSize = inputLayers_[0]->getOutputValue()->getHeight(); - resetOutput(batchSize, getSize()); + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + resetOutput(batchSize, getOutputSize()); MatrixPtr image = nullptr; - for (size_t i = 0; i != inputLayers_.size(); ++i) { + MatrixPtr outV = getOutputValue(); + for (size_t i = 0; i < inputLayers_.size(); ++i) { LayerPtr prevLayer = getPrev(i); image = prevLayer->getOutputValue(); for (size_t off = 0; off < image->getHeight(); off++) { REGISTER_TIMER_INFO("expandFwdOnce", getName().c_str()); - expandFwdOnce(image, i, off); + expandFwdOnce(image, outV, i, off); } } /* add the bias-vector */ - if (biases_.get() != NULL) { + if (biases_.get()) { if (sharedBiases_) { addSharedBias(); } else { @@ -182,29 +58,6 @@ void ExpandConvLayer::forward(PassType passType) { forwardActivation(); } -void ExpandConvLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { - size_t mapW = getSize() / numFilters_; - size_t mapH = v->getElementCnt() / mapW; - MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - vTmp->transpose(transOutValue_, false); // false means no memory allocation - vTmp->reshape(transOutValue_->getElementCnt() / numFilters_, numFilters_); - biases->collectBias(*vTmp, 1.0f); -} - -void ExpandConvLayer::bpropBiases(MatrixPtr v) { - MatrixPtr biases = - Matrix::create(biases_->getWGrad()->getData(), 1, - biases_->getWGrad()->getElementCnt(), false, useGpu_); - if (sharedBiases_) { - bpropSharedBias(biases, v); - } else { - biases->collectBias(*v, 1.0f); - } - biases->clear(); -} void ExpandConvLayer::backward(const UpdateCallback &callback) { backwardActivation(); @@ -216,111 +69,18 @@ void ExpandConvLayer::backward(const UpdateCallback &callback) { biases_->getParameterPtr()->incUpdate(callback); } - for (size_t i = 0; i != inputLayers_.size(); ++i) { + for (size_t i = 0; i < inputLayers_.size(); ++i) { /* First, calculate the input layers error */ - bpropActs(outGrad, i); + if (getPrev(i)->getOutputGrad()) { + bpropActs(outGrad, getPrev(i)->getOutputGrad(), i); + } if (weights_[i]->getWGrad()) { /* Then, calculate the W-gradient for the current layer */ - bpropWeights(outGrad, i); + bpropWeights(getPrev(i)->getOutputValue(), outGrad, i); /* Increasing the number of gradient */ weights_[i]->getParameterPtr()->incUpdate(callback); } } } -void ExpandConvLayer::bpropWeights(MatrixPtr v, int inpIdx) { - MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); - MatrixPtr inputV = getPrev(inpIdx)->getOutputValue(); - - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - size_t batchSize = inputV->getHeight(); - resetExpandInput(subK * groups_[inpIdx], subN); - resetConvOutput(batchSize, inpIdx); - - real *gradData = v->getData(); - - for (size_t n = 0; n < batchSize; n++) { // frame by frame - // expand - expandOneFrame(inputV, n, inpIdx); - real *wGradData = weightGrad->getData(); - real *expandInData = expandInput_->getData(); - - // expand-mul one-group by one - for (int g = 0; g < groups_[inpIdx]; g++) { - MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(gradData, subM, subN, true, useGpu_); - MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - gradData += subM * subN; - wGradData += subK * subM; - expandInData += subK * subN; - } - } -} - -void ExpandConvLayer::bpropActs(MatrixPtr v, int inpIdx) { - LayerPtr prevLayer = getPrev(inpIdx); - if (NULL == prevLayer->getOutputGrad()) { - return; - } - - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - size_t batchSize = v->getHeight(); - MatrixPtr tgtGrad = prevLayer->getOutputGrad(); - - /* reset the expand-grad memory */ - resetExpandInput(subK * groups_[inpIdx], subN); - resetConvOutput(batchSize, inpIdx); - - real *localGradData = v->getData(); - real *tgtGradData = tgtGrad->getData(); - for (size_t n = 0; n < batchSize; n++) { - real *wgtData = weights_[inpIdx]->getW()->getData(); - real *expandInData = expandInput_->getData(); - - for (int g = 0; g < groups_[inpIdx]; g++) { - // create temporary matrix - MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(localGradData, subM, subN, false, useGpu_); - MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); - C->mul(A, B); // mul - - // clear the temporary matrix - A->clear(); - B->clear(); - C->clear(); - - expandInData += subK * subN; - localGradData += subM * subN; - wgtData += subK * subM; - } - - // shrink one frame outGrad - MatrixPtr oneGradTmp = Matrix::create( - expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); - MatrixPtr vTmp = Matrix::create( - tgtGradData, 1, - imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channels_[inpIdx], false, - useGpu_); - vTmp->convShrink(*oneGradTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], - channels_[inpIdx], filterSize_[inpIdx], - filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], - padding_[inpIdx], padding_[inpIdx], - outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); - vTmp->clear(); - oneGradTmp->clear(); - - // move the data-pointer - tgtGradData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channels_[inpIdx]; - } -} - } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvLayer.h b/paddle/gserver/layers/ExpandConvLayer.h index fc3d69b1b7d14..c07188a406183 100644 --- a/paddle/gserver/layers/ExpandConvLayer.h +++ b/paddle/gserver/layers/ExpandConvLayer.h @@ -15,9 +15,9 @@ limitations under the License. */ #pragma once -#include "ConvBaseLayer.h" #include "paddle/math/Matrix.h" #include +#include "ExpandConvBaseLayer.h" namespace paddle { @@ -28,73 +28,18 @@ namespace paddle { * * The config file api is img_conv_layer. */ -class ExpandConvLayer : public ConvBaseLayer { -protected: - /// For expand convolution. - /// subM_ = numFilters_ / groups_. - IntV subM_; - /// subN_ = outputH_ * outputW_. - IntV subN_; - /// subK_ = channels_ * filterPixels_ * groups_. - IntV subK_; - /// The spatial dimensions of height of input feature map. - IntV imgSizeH_; - /// The spatial dimensions of width of input feature map. - IntV imgSizeW_; - /// The spatial dimensions of height of output feature map. - IntV outputH_; - /// The spatial dimensions of width of output feature map. - IntV outputW_; - /// Expand one sample at a time. shape: - /// (numChannels * filterPixels_, outputSizeH * outputSizeW) - MatrixPtr expandInput_; - /// The transpose of output, which is an auxiliary matrix. - MatrixPtr transOutValue_; +class ExpandConvLayer : public ExpandConvBaseLayer { public: - explicit ExpandConvLayer(const LayerConfig& config) : ConvBaseLayer(config) {} + explicit ExpandConvLayer(const LayerConfig& config) : + ExpandConvBaseLayer(config) {} ~ExpandConvLayer() {} bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - size_t getSize(); - - /** - * Create or resize expandInput_. - */ - void resetExpandInput(size_t height, size_t width); - - /** - * Create or resize transOutValue_. - */ - void resetConvOutput(size_t batchSize, int inIdx); - - /** - * Expand one input sample. - */ - void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); - - /** - * Expand one input sample and perform matrix multiplication. - */ - void expandFwdOnce(MatrixPtr image, int inIdx, int startIdx); - - /** - * Add shared bias. - */ - void addSharedBias(); - - /** - * Add unshared bias. - */ - void addUnsharedBias(); void forward(PassType passType); - void bpropSharedBias(MatrixPtr biases, MatrixPtr v); - void bpropBiases(MatrixPtr v); void backward(const UpdateCallback& callback); - void bpropWeights(MatrixPtr v, int inpIdx); - void bpropActs(MatrixPtr v, int inpIdx); }; } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp new file mode 100644 index 0000000000000..a3e160f1f4eb5 --- /dev/null +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -0,0 +1,92 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "paddle/utils/Logging.h" +#include "paddle/utils/Stat.h" +#include "ExpandConvTransLayer.h" + +/* The implementation of the convTransLayer is basically a swap of forward and + * backward of the original convLayer. + * The variable naming follows the convention of the convLayer. + * */ + +namespace paddle { + +REGISTER_LAYER(exconvt, ExpandConvTransLayer); + +bool ExpandConvTransLayer::init(const LayerMap &layerMap, + const ParameterMap ¶meterMap) { + /* Initialize the basic convolutional parent class */ + ExpandConvBaseLayer::init(layerMap, parameterMap); + + return true; +} + +void ExpandConvTransLayer::forward(PassType passType) { + Layer::forward(passType); + + /* malloc memory for the output_ if necessary */ + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + resetOutput(batchSize, getOutputSize()); + + MatrixPtr output = nullptr; + for (size_t i = 0; i < inputLayers_.size(); ++i) { + LayerPtr prevLayer = getPrev(i); + output = prevLayer->getOutputValue(); + REGISTER_TIMER_INFO("shrinkFwd", getName().c_str()); + bpropActs(output, getOutputValue(), i); + } + + /* add the bias-vector */ + if (biases_.get()) { + if (sharedBiases_) { + addSharedBias(); + } else { + addUnsharedBias(); + } + } + + /* activation */ + forwardActivation(); +} + +void ExpandConvTransLayer::backward(const UpdateCallback &callback) { + backwardActivation(); + + MatrixPtr imageGrad = getOutputGrad(); + if (biases_ && biases_->getWGrad()) { + bpropBiases(imageGrad); + /* Increasing the number of gradient */ + biases_->getParameterPtr()->incUpdate(callback); + } + + for (size_t i = 0; i < inputLayers_.size(); ++i) { + /* First, calculate the input layers error */ + for (size_t off = 0; off < imageGrad->getHeight(); off++) { + if (getPrev(i)->getOutputGrad()) { + expandFwdOnce(imageGrad, getPrev(i)->getOutputGrad(), i, off); + } + } + if (weights_[i]->getWGrad()) { + /* Then, calculate the W-gradient for the current layer */ + bpropWeights(imageGrad, getPrev(i)->getOutputValue(), i); + /* Increasing the number of gradient */ + weights_[i]->getParameterPtr()->incUpdate(callback); + } + } +} + + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h new file mode 100644 index 0000000000000..87c464a97f2ed --- /dev/null +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -0,0 +1,44 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "paddle/math/Matrix.h" +#include +#include "ExpandConvBaseLayer.h" + +namespace paddle { + +/** + * @brief A subclass of convolution layer. + * This layer expands input and use matrix multiplication to + * calculate convolution transpose (deconv) operation. + * + * The config file api is img_conv_layer with flag trans=True. + */ +class ExpandConvTransLayer : public ExpandConvBaseLayer { +public: + explicit ExpandConvTransLayer(const LayerConfig& config) : + ExpandConvBaseLayer(config) {} + + ~ExpandConvTransLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + void backward(const UpdateCallback& callback); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandLayer.cpp b/paddle/gserver/layers/ExpandLayer.cpp index bbd0b53273b43..9290ce4f6d46c 100644 --- a/paddle/gserver/layers/ExpandLayer.cpp +++ b/paddle/gserver/layers/ExpandLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "ExpandLayer.h" #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" @@ -53,9 +52,8 @@ void ExpandLayer::forward(PassType passType) { const Argument& shapeInput = getInput(1); const Argument& dataInput = getInput(0); size_t outputBatchSize = shapeInput.getBatchSize(); - auto startPositions = - type_ ? shapeInput.subSequenceStartPositions - : shapeInput.sequenceStartPositions; + auto startPositions = type_ ? shapeInput.subSequenceStartPositions + : shapeInput.sequenceStartPositions; size_t numSequences = startPositions->getSize() - 1; const int* starts = startPositions->getData(false); @@ -71,8 +69,7 @@ void ExpandLayer::forward(PassType passType) { // set output sequence info as shape sequence output_.sequenceStartPositions = shapeInput.sequenceStartPositions; if (shapeInput.hasSubseq()) { - output_.subSequenceStartPositions = - shapeInput.subSequenceStartPositions; + output_.subSequenceStartPositions = shapeInput.subSequenceStartPositions; } // reserve output: Expand output to batchsize of sequence data. @@ -81,8 +78,8 @@ void ExpandLayer::forward(PassType passType) { MatrixPtr inputValue = getInputValue(0); MatrixPtr outputValue = getOutputValue(); - IVector::resizeOrCreate(cpuExpandStartsPos_, outputBatchSize, false); - int* expandStarts = cpuExpandStartsPos_->getData(); + ICpuGpuVector::resizeOrCreate(expandStartsPos_, outputBatchSize, false); + int* expandStarts = expandStartsPos_->getMutableData(false); for (size_t sequenceId = 0; sequenceId < numSequences; ++sequenceId) { int sequenceLength = starts[sequenceId + 1] - starts[sequenceId]; for (int j = 0; j < sequenceLength; j++) { @@ -90,15 +87,8 @@ void ExpandLayer::forward(PassType passType) { } } - if (useGpu_) { - // TODO(Dangqingqing) move copyFrom - IVector::resizeOrCreate(expandStartsPos_, outputBatchSize, true); - expandStartsPos_->copyFrom(*cpuExpandStartsPos_, HPPL_STREAM_DEFAULT); - } else { - expandStartsPos_ = cpuExpandStartsPos_; - } - - outputValue->copyByRowIndex(*inputValue, *expandStartsPos_); + outputValue->copyByRowIndex(*inputValue, + *expandStartsPos_->getVector(useGpu_)); if (biases_.get() != NULL) { outputValue->addBias(*(biases_->getW()), 1); @@ -108,16 +98,15 @@ void ExpandLayer::forward(PassType passType) { void ExpandLayer::backward(const UpdateCallback& callback) { if (biases_ && biases_->getWGrad()) { biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - /* Increasing the number of gradient */ + /* Increasing the number of gradient */ biases_->getParameterPtr()->incUpdate(callback); } if (!getInputGrad(0)) return; MatrixPtr inputGrad = getInputGrad(0); MatrixPtr outputGrad = getOutputGrad(); - auto cpuSeqStartPos = - type_ ? getInput(1).subSequenceStartPositions - : getInput(1).sequenceStartPositions; + auto cpuSeqStartPos = type_ ? getInput(1).subSequenceStartPositions + : getInput(1).sequenceStartPositions; size_t numSequences = cpuSeqStartPos->getSize() - 1; const int* starts = cpuSeqStartPos->getData(false); diff --git a/paddle/gserver/layers/ExpandLayer.h b/paddle/gserver/layers/ExpandLayer.h index 8a3eb1c973a47..fbe0ced9b1754 100644 --- a/paddle/gserver/layers/ExpandLayer.h +++ b/paddle/gserver/layers/ExpandLayer.h @@ -44,14 +44,9 @@ class ExpandLayer : public Layer { enum ExpandLevel { kNonSeq = 0, kSeq = 1 }; /// store the ExpandLevel int type_; - // TODO(luotao) use ICpuGpuVectorPtr to merge cpuExpandStartsPos_ - // and expandStartsPos_ /// expanded sequenceStartPositions or subSequenceStartPositions /// of input[1] - IVectorPtr cpuExpandStartsPos_; - /// point to cpuExpandStartsPos_ when useGpu_ is false, - /// copy from cpuExpandStartsPos_ when useGpu_ is true - IVectorPtr expandStartsPos_; + ICpuGpuVectorPtr expandStartsPos_; public: explicit ExpandLayer(const LayerConfig& config) : Layer(config) {} diff --git a/paddle/gserver/layers/FullMatrixProjection.cpp b/paddle/gserver/layers/FullMatrixProjection.cpp index 8241cbd37ec62..f17c1b05bd892 100644 --- a/paddle/gserver/layers/FullMatrixProjection.cpp +++ b/paddle/gserver/layers/FullMatrixProjection.cpp @@ -52,7 +52,9 @@ void FullMatrixProjection::backward(const UpdateCallback& callback) { } hl_set_sync_flag(syncFlag); - parameter_->incUpdate(callback); + if (weight_->getWGrad()) { + parameter_->incUpdate(callback); + } } } // namespace paddle diff --git a/paddle/gserver/layers/FullyConnectedLayer.h b/paddle/gserver/layers/FullyConnectedLayer.h index 24b6c547e7bc8..334eb4b722f4f 100644 --- a/paddle/gserver/layers/FullyConnectedLayer.h +++ b/paddle/gserver/layers/FullyConnectedLayer.h @@ -48,4 +48,3 @@ class FullyConnectedLayer : public Layer { }; } // namespace paddle - diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp index 44ea95c80ab08..78d15c553021d 100644 --- a/paddle/gserver/layers/Layer.cpp +++ b/paddle/gserver/layers/Layer.cpp @@ -16,6 +16,7 @@ limitations under the License. */ #include "paddle/utils/Util.h" #include "paddle/utils/Logging.h" +#include "paddle/math/SparseMatrix.h" #include "AddtoLayer.h" #include "CosSimLayer.h" @@ -290,14 +291,30 @@ void Layer::showOutputStats() { << " is 0, skip to show the statistics"; return; } - real mean = out->getSum() / out->getElementCnt(); - MatrixPtr outSquare = out->clone(); - outSquare->copyFrom(*out); + MatrixPtr outSquare; + if (dynamic_cast(out.get())) { + GpuSparseMatrix *tmp = dynamic_cast(out.get()); + outSquare = std::make_shared( + tmp->getHeight(), tmp->getWidth(), tmp->getElementCnt(), + tmp->getValueType(), tmp->getFormat()); + } else { + outSquare = out->clone(); + } + outSquare->copyFrom(*out, HPPL_STREAM_DEFAULT); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); + + real mean = outSquare->getSum() / out->getElementCnt(); + real min; + real max; if (dynamic_cast(outSquare.get())) { auto tmpMat = dynamic_cast(outSquare.get()); + min = tmpMat->getMin(); + max = tmpMat->getMax(); tmpMat->square(); LOG(INFO) << "show statistics of [none zero values] in sparse matrix"; } else { + min = outSquare->getMin(); + max = outSquare->getMax(); outSquare->square(); } real std = (outSquare->getSum() / outSquare->getElementCnt()) - mean * mean; @@ -306,8 +323,8 @@ void Layer::showOutputStats() { << ", " << "std=" << std << ", " - << "min=" << out->getMin() << ", " - << "max=" << out->getMax(); + << "min=" << min << ", " + << "max=" << max; } void Layer::forwardActivation() { diff --git a/paddle/gserver/layers/MaxLayer.cpp b/paddle/gserver/layers/MaxLayer.cpp index 226e0ea87dbd4..c4ffe894eccd6 100644 --- a/paddle/gserver/layers/MaxLayer.cpp +++ b/paddle/gserver/layers/MaxLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "MaxLayer.h" #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" @@ -21,55 +20,11 @@ namespace paddle { REGISTER_LAYER(max, MaxLayer); -bool MaxLayer::init(const LayerMap& layerMap, - const ParameterMap& parameterMap) { - /* Initialize the basic parent class */ - Layer::init(layerMap, parameterMap); - - /* initialize biases_ */ - if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); - } - - // transform to which sequence type - if (config_.trans_type() == "non-seq") { - type_ = kNonSeq; - } else if (config_.trans_type() == "seq") { - type_ = kSeq; - } else { - LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); - } - setNeedSequenceInfo(false); - return true; -} - void MaxLayer::forward(PassType passType) { - Layer::forward(passType); - // max layer should have exactly 1 input - CHECK_EQ(1U, inputLayers_.size()); - - size_t dim = getSize(); - const Argument& input = getInput(0); - int64_t newBatchSize = - type_ ? input.getNumSubSequences() : input.getNumSequences(); - ICpuGpuVectorPtr startPositions = - type_ ? input.subSequenceStartPositions - : input.sequenceStartPositions; - auto starts = startPositions->getVector(useGpu_); - size_t numSequences = startPositions->getSize() - 1; + SequencePoolLayer::forward(passType); - CHECK_EQ(dim, input.value->getWidth()); - CHECK_EQ(numSequences, (size_t)newBatchSize); - CHECK_EQ(startPositions->getData(false)[numSequences], input.getBatchSize()); - if (type_) { - // when trans_type = seq, input must hasSubseq - CHECK_EQ(input.hasSubseq(), 1UL); - } - - // reset output: resize to "num of sequences", not "batch size". - resetOutput(newBatchSize, dim); - - IVector::resizeOrCreate(maxIndex_, newBatchSize * dim, useGpu(deviceId_)); + IVector::resizeOrCreate(maxIndex_, newBatchSize_ * getSize(), + useGpu(deviceId_)); maxIndex_->zeroMem(); MatrixPtr inputValue = getInputValue(0); @@ -77,16 +32,8 @@ void MaxLayer::forward(PassType passType) { { REGISTER_TIMER_INFO("MaxLayerForward", getName().c_str()); - outputValue->maxSequenceForward(*inputValue, *starts, *maxIndex_); - } - - /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, - * thus, in this case, output_ has no cpuSequenceStartPositions. - * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this - * case, we should compute the new cpuSequenceStartPositions. - */ - if (type_) { - output_.degradeSequence(input, useGpu_); + outputValue->maxSequenceForward( + *inputValue, *startPositions_->getVector(useGpu_), *maxIndex_); } if (config_.output_max_index()) { @@ -104,24 +51,14 @@ void MaxLayer::forward(PassType passType) { void MaxLayer::backward(const UpdateCallback& callback) { CHECK(!config_.output_max_index()) << "backward is not available when output_max_index is set"; - /* Do derivation */ { backwardActivation(); } - - if (biases_ && biases_->getWGrad()) { - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - - // Increasing the number of gradient - biases_->getParameterPtr()->incUpdate(callback); - } + SequencePoolLayer::backward(callback); MatrixPtr inputGrad = getInputGrad(0); MatrixPtr outputGrad = getOutputGrad(); if (inputGrad) { - ICpuGpuVectorPtr starts = - type_ ? getInput(0).subSequenceStartPositions - : getInput(0).sequenceStartPositions; REGISTER_TIMER_INFO("MaxLayerBackward", getName().c_str()); - inputGrad->maxSequenceBackward(*outputGrad, - *(starts->getVector(useGpu_)), *maxIndex_); + inputGrad->maxSequenceBackward( + *outputGrad, *(startPositions_->getVector(useGpu_)), *maxIndex_); } } diff --git a/paddle/gserver/layers/MaxLayer.h b/paddle/gserver/layers/MaxLayer.h index b4c34e665d926..e6dcfe9c6759d 100644 --- a/paddle/gserver/layers/MaxLayer.h +++ b/paddle/gserver/layers/MaxLayer.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once -#include "Layer.h" +#include "SequencePoolLayer.h" #include "paddle/math/Matrix.h" #include "paddle/utils/ThreadLocal.h" @@ -24,29 +24,30 @@ namespace paddle { /** * A layer for "internal max" for sequence input. * Input: one or more sequences. Each sequence contains some instances. - * If MaxLevel = kNonSeq: + * If SequenceLevel = kNonSeq: * Output: output size is the number of input sequences (NOT input instances) * output[i] = max_{for each instance in this sequence}{input[i]} - * If MaxLevel = kSeq: + * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: output size is the number of input sub-sequences * output[i] = max_{for each instance in this sub-sequence}{input[i]} + * + * The config file api is pooling_layer. */ -class MaxLayer : public Layer { +class MaxLayer : public SequencePoolLayer { protected: - std::unique_ptr biases_; // maxIndex_[i][j] = k : the value at (i, j) is from input[k]. IVectorPtr maxIndex_; - int type_; public: - explicit MaxLayer(const LayerConfig& config) : Layer(config) {} - enum MaxLevel {kNonSeq = 0, kSeq = 1 }; + explicit MaxLayer(const LayerConfig& config) : SequencePoolLayer(config) {} ~MaxLayer() {} - bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap) { + return SequencePoolLayer::init(layerMap, parameterMap); + } void forward(PassType passType); void backward(const UpdateCallback& callback = nullptr); diff --git a/paddle/gserver/layers/MaxOutLayer.cpp b/paddle/gserver/layers/MaxOutLayer.cpp new file mode 100644 index 0000000000000..a3de069bf7a6c --- /dev/null +++ b/paddle/gserver/layers/MaxOutLayer.cpp @@ -0,0 +1,87 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "MaxOutLayer.h" +#include "hl_gpu.h" +#include "hl_cnn.h" + +namespace paddle { + +REGISTER_LAYER(maxout, MaxOutLayer); + +size_t MaxOutLayer::getSize() { + const MaxOutConfig& maxoutConf = config_.inputs(0).maxout_conf(); + imgSizeH_ = inputLayers_[0]->getOutput().getFrameHeight(); + imgSizeW_ = inputLayers_[0]->getOutput().getFrameWidth(); + if (imgSizeH_ == 0) { + imgSizeH_ = maxoutConf.img_size_y(); + } + if (imgSizeW_ == 0) { + imgSizeW_ = maxoutConf.img_size_x(); + } + + featLen_ = imgSizeH_ * imgSizeW_; + size_t layerSize = featLen_ * outputChannels_; + + getOutput().setFrameHeight(imgSizeH_); + getOutput().setFrameWidth(imgSizeW_); + + return layerSize; +} + +bool MaxOutLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + /* the size of inputs for maxout-layer is 1 */ + CHECK_EQ(config_.inputs_size(), 1); + + const MaxOutConfig& conf = config_.inputs(0).maxout_conf(); + groups_ = conf.groups(); + channels_ = conf.channels(); + CHECK_EQ(channels_ % groups_, 0UL); + outputChannels_ = channels_ / groups_; + + return true; +} + +void MaxOutLayer::forward(PassType passType) { + Layer::forward(passType); + + /* malloc memory for the output_ if necessary */ + /* note: one sample correspond to one column */ + size_t batchSize = getInput(0).getBatchSize(); + size_t size = getSize(); + resetOutput(batchSize, size); + MatrixPtr inputV = getInputValue(0); + MatrixPtr outV = getOutputValue(); + + IVector::resizeOrCreate(maxoutId_, size * batchSize, useGpu_); + outV->maxoutForward(*inputV, *maxoutId_, outputChannels_, groups_); +} + +void MaxOutLayer::backward(const UpdateCallback& callback) { + (void)callback; + + /* Do derivation */ + MatrixPtr inputG = getInputGrad(0); + MatrixPtr outG = getOutputGrad(); + + if (inputG) { + inputG->maxoutBackward(*outG, *maxoutId_, outputChannels_, groups_); + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/MaxOutLayer.h b/paddle/gserver/layers/MaxOutLayer.h new file mode 100644 index 0000000000000..9011a5c332b17 --- /dev/null +++ b/paddle/gserver/layers/MaxOutLayer.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "paddle/math/Matrix.h" + +namespace paddle { + +/** + * A layer to do max out on conv layer output. + * Input: output of a conv layer. + * Output: feature map size same as input. Channel is (input channel) / groups. + * So the num of channels should be able to devided by groups. + * + * The config file api is maxout_layer. + */ + +class MaxOutLayer : public Layer { +protected: + size_t groups_; + size_t imgSizeH_, imgSizeW_; + /// outputChannels_ = channels_ / groups_ + size_t channels_, outputChannels_; + /// feature length = imgSizeH_ * imgSizeW_ + size_t featLen_; + IVectorPtr maxoutId_; + +public: + /// return imgSizeH_ * imgSizeW_ * outputChannels_; + size_t getSize(); + + explicit MaxOutLayer(const LayerConfig& config) : Layer(config) {} + virtual ~MaxOutLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + void backward(const UpdateCallback& callback = nullptr); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/MixedLayer.cpp b/paddle/gserver/layers/MixedLayer.cpp index 054ddd3a228ed..26b1360290ffb 100644 --- a/paddle/gserver/layers/MixedLayer.cpp +++ b/paddle/gserver/layers/MixedLayer.cpp @@ -41,9 +41,13 @@ bool MixedLayer::init(const LayerMap& layerMap, } operators_.emplace_back(Operator::create(operator_conf, useGpu_)); } + /* initialize biases_ */ if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); + sharedBias_ = config_.shared_biases(); + size_t psize = config_.bias_size(); + biases_ = std::unique_ptr( + new Weight(1, psize, biasParameter_)); } return true; @@ -119,12 +123,6 @@ void MixedLayer::forward(PassType passType) { MatrixPtr outV = getOutputValue(); - /* add the bias-vector */ - if (biases_.get() != NULL) { - REGISTER_TIMER_INFO("FwBiasTimer", getName().c_str()); - outV->addBias(*(biases_->getW()), 1); - } - for (size_t i = 0; i != inputLayers_.size(); ++i) { if (projections_[i]) { projections_[i]->forward(&getInput(i), &output_, passType); @@ -140,6 +138,12 @@ void MixedLayer::forward(PassType passType) { op->forward(ins, &output_, passType); } + /* add the bias-vector */ + if (biases_.get() != NULL) { + REGISTER_TIMER_INFO("FwBiasTimer", getName().c_str()); + outV->addBias(*(biases_->getW()), 1, sharedBias_); + } + /* activation */ { REGISTER_TIMER_INFO("FwAtvTimer", getName().c_str()); forwardActivation(); @@ -154,7 +158,7 @@ void MixedLayer::backward(const UpdateCallback& callback) { if (biases_ && biases_->getWGrad()) { REGISTER_TIMER_INFO("BpBiasTimer", getName().c_str()); - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); + biases_->getWGrad()->collectBias(*getOutputGrad(), 1, sharedBias_); /* Increasing the number of gradient */ biases_->getParameterPtr()->incUpdate(callback); diff --git a/paddle/gserver/layers/MixedLayer.h b/paddle/gserver/layers/MixedLayer.h index 9bac1355bd21f..5842e51e1d79d 100644 --- a/paddle/gserver/layers/MixedLayer.h +++ b/paddle/gserver/layers/MixedLayer.h @@ -58,5 +58,6 @@ class MixedLayer : public Layer { /// the matrix size of projection state std::vector projectionStateMatrixSize_; std::unique_ptr biases_; + bool sharedBias_; }; } // namespace paddle diff --git a/paddle/gserver/layers/NCELayer.cpp b/paddle/gserver/layers/NCELayer.cpp index a896e16a6027b..4faebe5d2ad6f 100644 --- a/paddle/gserver/layers/NCELayer.cpp +++ b/paddle/gserver/layers/NCELayer.cpp @@ -21,14 +21,18 @@ limitations under the License. */ namespace paddle { /** - * Noise-contrastive estimation + * Noise-contrastive estimation. * Implements the method in the following paper: - * A fast and simple algorithm for training neural probabilistic language models + * A fast and simple algorithm for training neural probabilistic language models. + * + * The config file api is nce_layer. */ class NCELayer : public Layer { int numClasses_; - int numInputs_; // number of input layer besides labelLayer and weightLayer + /// number of input layer besides labelLayer and weightLayer + int numInputs_; LayerPtr labelLayer_; + /// weight layer, can be None LayerPtr weightLayer_; WeightList weights_; std::unique_ptr biases_; @@ -43,7 +47,8 @@ class NCELayer : public Layer { real weight; }; std::vector samples_; - bool prepared_; // whether samples_ is prepared + /// whether samples_ is prepared + bool prepared_; Argument sampleOut_; IVectorPtr labelIds_; diff --git a/paddle/gserver/layers/PoolLayer.cpp b/paddle/gserver/layers/PoolLayer.cpp index 0ff7f374abb4b..2fbc9001f1161 100644 --- a/paddle/gserver/layers/PoolLayer.cpp +++ b/paddle/gserver/layers/PoolLayer.cpp @@ -35,7 +35,6 @@ bool PoolLayer::init(const LayerMap& layerMap, poolType_ = conf.pool_type(); channels_ = conf.channels(); sizeX_ = conf.size_x(); - start_ = conf.start(); stride_ = conf.stride(); outputX_ = conf.output_x(); imgSize_ = conf.img_size(); @@ -47,32 +46,14 @@ bool PoolLayer::init(const LayerMap& layerMap, confPaddingY_ = conf.has_padding_y() ? conf.padding_y() : conf.padding(); outputY_ = conf.has_output_y() ? conf.output_y() : conf.output_x(); - bool cudnnTypeCheck = true; -#ifndef PADDLE_ONLY_CPU - cudnnTypeCheck = !CudnnPoolLayer::typeCheck(poolType_); -#endif - - if ((sizeY_ != sizeX_ || imgSizeY_ != imgSize_ || strideY_ != stride_ || - confPaddingY_ != confPadding_ || outputY_ != outputX_) && - cudnnTypeCheck) { - LOG(FATAL) << poolType_ << " does not supported non-square " - "filter, image, stride or padding"; - } - - if (confPadding_ != 0 && cudnnTypeCheck) { - LOG(FATAL) << poolType_ << " does not supported 'padding'"; - } - return true; } Layer* PoolLayer::create(const LayerConfig& config) { CHECK_EQ(config.inputs_size(), 1); const std::string& pool = config.inputs(0).pool_conf().pool_type(); - if (pool == "max-projection") { - return new MaxPoolProjectionLayer(config); - } else if (pool == "avg-projection") { - return new AvgPoolProjectionLayer(config); + if (pool == "max-projection" || pool == "avg-projection") { + return new PoolProjectionLayer(config); #ifndef PADDLE_ONLY_CPU } else if (CudnnPoolLayer::typeCheck(pool)) { return new CudnnPoolLayer(config); diff --git a/paddle/gserver/layers/PoolLayer.h b/paddle/gserver/layers/PoolLayer.h index b7a1dfd7632f9..e87ad08251dd4 100644 --- a/paddle/gserver/layers/PoolLayer.h +++ b/paddle/gserver/layers/PoolLayer.h @@ -17,6 +17,7 @@ limitations under the License. */ #include "Layer.h" #include "paddle/math/Matrix.h" +#include "paddle/math/MathUtils.h" #include namespace paddle { @@ -28,7 +29,7 @@ namespace paddle { class PoolLayer : public Layer { protected: size_t channels_, sizeX_, stride_, outputX_, imgSize_; - int start_, confPadding_; + int confPadding_; size_t sizeY_; size_t imgSizeY_; diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp new file mode 100644 index 0000000000000..9be5aba3d57d2 --- /dev/null +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -0,0 +1,123 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "PoolProjection.h" + +namespace paddle { + +REGISTER_PROJECTION_CREATE_FUNC(pool, &PoolProjection::create); + +PoolProjection::PoolProjection(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) + : Projection(config, parameter, useGpu) { + const PoolConfig& conf = config_.pool_conf(); + poolType_ = conf.pool_type(); + channels_ = conf.channels(); + sizeX_ = conf.size_x(); + stride_ = conf.stride(); + outputX_ = conf.output_x(); + imgSize_ = conf.img_size(); + confPadding_ = conf.padding(); + + sizeY_ = conf.has_size_y() ? conf.size_y() : conf.size_x(); + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + strideY_ = conf.has_stride_y() ? conf.stride_y() : conf.stride(); + confPaddingY_ = conf.has_padding_y() ? conf.padding_y() : conf.padding(); + outputY_ = conf.has_output_y() ? conf.output_y() : conf.output_x(); +} + +size_t PoolProjection::getSize() { + imgSizeY_ = in_->getFrameHeight(); + imgSize_ = in_->getFrameWidth(); + const PoolConfig& conf = config_.pool_conf(); + if (imgSizeY_ == 0) { + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + } + if (imgSize_ == 0) { + imgSize_ = conf.img_size(); + } + outputY_ = outputSize(imgSizeY_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputX_ = outputSize(imgSize_, sizeX_, confPadding_, stride_, + /* caffeMode */ false); + + const_cast(out_)->setFrameHeight(outputY_); + const_cast(out_)->setFrameWidth(outputX_); + + return outputY_ * outputX_ * channels_; +} + +PoolProjection* PoolProjection::create(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) { + const std::string& pool = config.pool_conf().pool_type(); + if (pool == "max-projection") { + return new MaxPoolProjection(config, parameter, useGpu); + } else if (pool == "avg-projection") { + return new AvgPoolProjection(config, parameter, useGpu); + } else { + LOG(FATAL) << "Unknown pool type: " << pool; + return nullptr; + } +} + +void MaxPoolProjection::forward() { + size_t width = getSize(); + CHECK_EQ(width, out_->value->getWidth()); + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + outV->maxPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, confPaddingY_, + confPadding_); +} + +void MaxPoolProjection::backward(const UpdateCallback& callback) { + (void)callback; + MatrixPtr outGrad = out_->grad; + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + MatrixPtr inputGrad = in_->grad; + + if (NULL == inputGrad) { + return; + } + inputGrad->maxPoolBackward(*inputV, imgSizeY_, imgSize_, *outGrad, *outV, + sizeX_, sizeY_, strideY_, stride_, outputY_, + outputX_, 1, 1, confPaddingY_, confPadding_); +} + +void AvgPoolProjection::forward() { + size_t width = getSize(); + CHECK_EQ(width, out_->value->getWidth()); + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + outV->avgPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, confPaddingY_, + confPadding_); +} + +void AvgPoolProjection::backward(const UpdateCallback& callback) { + (void)callback; + + MatrixPtr outputGrad = out_->grad; + MatrixPtr inputGrad = in_->grad; + + if (NULL == inputGrad) { + return; + } + + inputGrad->avgPoolBackward(*outputGrad, imgSizeY_, imgSize_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, 1, 1, + confPaddingY_, confPadding_); +} +} // namespace paddle diff --git a/paddle/gserver/layers/PoolProjection.h b/paddle/gserver/layers/PoolProjection.h new file mode 100644 index 0000000000000..a11e25b729cb7 --- /dev/null +++ b/paddle/gserver/layers/PoolProjection.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Projection.h" +#include "paddle/math/MathUtils.h" + +namespace paddle { + +class PoolProjection : public Projection { +protected: + size_t imgSizeY_, imgSize_; + size_t outputY_, outputX_; + size_t strideY_, stride_; + size_t sizeY_, sizeX_; + int confPaddingY_, confPadding_; + size_t channels_; + std::string poolType_; + +public: + PoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu); + + static PoolProjection* create(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu); + + const std::string& getPoolType() const { return poolType_; } + + size_t getSize(); +}; + +class MaxPoolProjection : public PoolProjection { +public: + MaxPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu) + : PoolProjection(config, parameter, useGpu) {} + + virtual void forward(); + virtual void backward(const UpdateCallback& callback = nullptr); +}; + +class AvgPoolProjection : public PoolProjection { +public: + AvgPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu) + : PoolProjection(config, parameter, useGpu) {} + + virtual void forward(); + virtual void backward(const UpdateCallback& callback = nullptr); +}; +} // namespace paddle diff --git a/paddle/gserver/layers/PoolProjectionLayer.cpp b/paddle/gserver/layers/PoolProjectionLayer.cpp index 9c2d6d2164a3f..cabb346d6c991 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.cpp +++ b/paddle/gserver/layers/PoolProjectionLayer.cpp @@ -12,92 +12,49 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" #include "PoolProjectionLayer.h" namespace paddle { + size_t PoolProjectionLayer::getSize() { CHECK_EQ(inputLayers_.size(), 1UL); size_t layerSize = 0; imgSizeH_ = inputLayers_[0]->getOutput().getFrameHeight(); imgSizeW_ = inputLayers_[0]->getOutput().getFrameWidth(); if (imgSizeH_ == 0) { - imgSizeH_ = imgSize_; + imgSizeH_ = imgSizeY_; } if (imgSizeW_ == 0) { imgSizeW_ = imgSize_; } - outputH_ = 1 + (imgSizeH_ - start_ - sizeX_ + stride_ - 1) / stride_; - outputW_ = 1 + (imgSizeW_ - start_ - sizeX_ + stride_ - 1) / stride_; - layerSize = outputH_ * outputW_ * channels_; - getOutput().setFrameHeight(outputH_); - getOutput().setFrameWidth(outputW_); - return layerSize; -} - -void MaxPoolProjectionLayer::forward(PassType passType) { - Layer::forward(passType); - - /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one ROW */ - MatrixPtr input = getInputValue(0); - int batchSize = input->getHeight(); - int size = getSize(); - resetOutput(batchSize, size); - - MatrixPtr outV = getOutputValue(); - - outV->maxPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, start_, - stride_, outputH_, outputW_); -} + outputH_ = outputSize(imgSizeH_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputW_ = outputSize(imgSizeW_, sizeX_, confPadding_, stride_, + /* caffeMode */ false); -void MaxPoolProjectionLayer::backward(const UpdateCallback& callback) { - (void)callback; - - if (NULL == getInputGrad(0)) { - return; - } - - /* Do derivation */ - MatrixPtr outGrad = getOutputGrad(); - MatrixPtr inputV = getInputValue(0); - MatrixPtr outV = getOutputValue(); - MatrixPtr inputGrad = getInputGrad(0); + layerSize = outputH_ * outputW_ * channels_; - inputGrad->maxPoolBackward(*inputV, imgSizeH_, imgSizeW_, *outGrad, *outV, - sizeX_, start_, stride_, outputH_, outputW_, 1, 1); + return layerSize; } -void AvgPoolProjectionLayer::forward(PassType passType) { +void PoolProjectionLayer::forward(PassType passType) { Layer::forward(passType); - - /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one ROW */ - MatrixPtr input = getInputValue(0); - int batchSize = input->getHeight(); + const Argument& in = getInput(0); + int batchSize = in.value->getHeight(); int size = getSize(); resetOutput(batchSize, size); - - MatrixPtr outV = getOutputValue(); - - outV->avgPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, start_, - stride_, outputH_, outputW_); + poolProjection_->forward(&in, &output_, passType); } -void AvgPoolProjectionLayer::backward(const UpdateCallback& callback) { +void PoolProjectionLayer::backward(const UpdateCallback& callback) { (void)callback; - if (NULL == getInputGrad(0)) { return; } - /* Do derivation */ - MatrixPtr outputGrad = getOutputGrad(); - MatrixPtr inputGrad = getInputGrad(0); - inputGrad->avgPoolBackward(*outputGrad, imgSizeH_, imgSizeW_, sizeX_, start_, - stride_, outputH_, outputW_, 1, 1); + poolProjection_->backward(callback); } } // namespace paddle diff --git a/paddle/gserver/layers/PoolProjectionLayer.h b/paddle/gserver/layers/PoolProjectionLayer.h index 42bbc83c62246..777b6f39e7cc4 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.h +++ b/paddle/gserver/layers/PoolProjectionLayer.h @@ -12,12 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once +#include #include "PoolLayer.h" +#include "PoolProjection.h" #include "paddle/math/Matrix.h" -#include namespace paddle { /** @@ -27,33 +27,18 @@ class PoolProjectionLayer : public PoolLayer { protected: size_t imgSizeH_, imgSizeW_; size_t outputH_, outputW_; + std::unique_ptr poolProjection_; + ProjectionConfig projectionConfig_; public: - size_t getSize(); - explicit PoolProjectionLayer(const LayerConfig& config) : PoolLayer(config) {} -}; -/** - * @brief A layer for max pooling - */ -class MaxPoolProjectionLayer : public PoolProjectionLayer { -public: - explicit MaxPoolProjectionLayer(const LayerConfig& config) - : PoolProjectionLayer(config) {} - - ~MaxPoolProjectionLayer() {} + explicit PoolProjectionLayer(const LayerConfig& config) : PoolLayer(config) { + PoolConfig* conf = projectionConfig_.mutable_pool_conf(); + *conf = config_.inputs(0).pool_conf(); + poolProjection_.reset( + PoolProjection::create(projectionConfig_, nullptr, useGpu_)); + } - virtual void forward(PassType passType); - virtual void backward(const UpdateCallback& callback = nullptr); -}; -/** - * @brief A layer for average pooling - */ -class AvgPoolProjectionLayer : public PoolProjectionLayer { -public: - explicit AvgPoolProjectionLayer(const LayerConfig& config) - : PoolProjectionLayer(config) {} - - ~AvgPoolProjectionLayer() {} + size_t getSize(); virtual void forward(PassType passType); virtual void backward(const UpdateCallback& callback = nullptr); diff --git a/paddle/gserver/layers/Projection.h b/paddle/gserver/layers/Projection.h index 3fa3a0cc230ac..203edc5396a53 100644 --- a/paddle/gserver/layers/Projection.h +++ b/paddle/gserver/layers/Projection.h @@ -12,12 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once -#include "paddle/parameter/Parameter.h" -#include "ModelConfig.pb.h" #include "Layer.h" +#include "ModelConfig.pb.h" +#include "paddle/parameter/Parameter.h" namespace paddle { @@ -28,6 +27,11 @@ namespace paddle { Projection::registrar_.registerClass<__class_name>(#__type_name); \ }) +#define REGISTER_PROJECTION_CREATE_FUNC(__type_name, createFunction) \ + static InitFunction __reg_type_##__type_name([]() { \ + Projection::registrar_.registerClass(#__type_name, createFunction); \ + }) + /** * A projection takes one Argument as input, calculate the result and add it * to output Argument. @@ -50,7 +54,8 @@ class Projection { registrar_; /** - * Forward propagation. If backward() will be called, in and out must be kept valid until then. + * Forward propagation. If backward() will be called, in and out must be kept + * valid until then. * @param in input of projection * @param out output of projection * @param passType PASS_TRAIN of PASS_TEST diff --git a/paddle/gserver/layers/ScalingProjection.cpp b/paddle/gserver/layers/ScalingProjection.cpp new file mode 100644 index 0000000000000..c0a7072c6a7cc --- /dev/null +++ b/paddle/gserver/layers/ScalingProjection.cpp @@ -0,0 +1,53 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "Projection.h" + +namespace paddle { + +class ScalingProjection : public Projection { +public: + ScalingProjection(const ProjectionConfig& config, + const ParameterPtr& parameter, bool useGpu) + : Projection(config, parameter, useGpu) { + CHECK_EQ(parameter->getSize(), 1UL); + weight_.reset(new Weight(1, 1, parameter)); + } + + void forward() { + CHECK(in_->value); + out_->value->add(*in_->value, weight_->getW()->getElement(0, 0)); + } + + void backward(const UpdateCallback& callback) { + if (weight_->getWGrad()) { + auto sum = Matrix::create(in_->value->getHeight(), 1, false, useGpu_); + sum->sumOfProducts(*in_->value, *out_->grad, + /* scaleSum= */1, /* scaleDest= */0); + weight_->getWGrad()->sumCols(*sum, + /* scaleSum= */1, /* scaleDest= */1); + parameter_->incUpdate(callback); + } + if (in_->grad) { + in_->grad->add(*out_->grad, weight_->getW()->getElement(0, 0)); + } + } + +protected: + std::unique_ptr weight_; +}; + +REGISTER_PROJECTION(scaling, ScalingProjection); + +} // namespace paddle diff --git a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp index 12831e3668802..26d9536dd57aa 100644 --- a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp +++ b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp @@ -15,7 +15,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" -#include "Layer.h" +#include "SequencePoolLayer.h" #include "paddle/math/Matrix.h" #include "paddle/utils/Stat.h" @@ -29,20 +29,19 @@ namespace paddle { * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: a sequence containing only the last instance of each sub-sequence - * of the input sequence + * of the input sequence + * + * The config file api is last_seq and first_seq. */ -class SequenceLastInstanceLayer : public Layer { +class SequenceLastInstanceLayer : public SequencePoolLayer { protected: - std::unique_ptr biases_; MatrixPtr tmpSrc_; MatrixPtr tmpDest_; - enum SequenceLevel { kNonSeq = 0, kSeq = 1 }; - int type_; public: explicit SequenceLastInstanceLayer(const LayerConfig& config) - : Layer(config) {} + : SequencePoolLayer(config) {} ~SequenceLastInstanceLayer() {} @@ -56,55 +55,20 @@ REGISTER_LAYER(seqlastins, SequenceLastInstanceLayer); bool SequenceLastInstanceLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { - /* Initialize the basic parent class */ - Layer::init(layerMap, parameterMap); - - // seqlastins layer should have exactly 1 input - CHECK_EQ(1U, inputLayers_.size()); - - /* initialize biases_ */ - if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); - } + SequencePoolLayer::init(layerMap, parameterMap); tmpSrc_ = Matrix::create(nullptr, /* height= */ 1, 1, /* trans= */ false, useGpu_); tmpDest_ = Matrix::create(nullptr, /* height= */ 1, 1, /* trans= */ false, useGpu_); - // transform to which sequence type - if (config_.trans_type() == "non-seq") { - type_ = kNonSeq; - } else if (config_.trans_type() == "seq") { - type_ = kSeq; - } else { - LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); - } - setNeedSequenceInfo(false); return true; } void SequenceLastInstanceLayer::forward(PassType passType) { - Layer::forward(passType); - - size_t dim = getSize(); - const Argument& input = getInput(0); - - // check - auto startPositions = - type_ ? input.subSequenceStartPositions->getVector(false) - : input.sequenceStartPositions->getVector(false); - size_t height = type_ ? input.getNumSubSequences() : input.getNumSequences(); - CHECK_EQ(dim, input.value->getWidth()); - CHECK_EQ(startPositions->getData()[height], input.getBatchSize()); - CHECK_EQ(height, startPositions->getSize() - 1); - if (type_) { - // when trans_type = seq, input must hasSubseq - CHECK_EQ(input.hasSubseq(), 1UL); - } + SequencePoolLayer::forward(passType); - reserveOutput(height, dim); - const int* starts = startPositions->getData(); + const int* starts = startPositions_->getData(false); MatrixPtr inputValue = getInputValue(0); MatrixPtr outputValue = getOutputValue(); @@ -112,21 +76,13 @@ void SequenceLastInstanceLayer::forward(PassType passType) { AsyncGpuBlock asyncGpuBlock; REGISTER_TIMER_INFO("SequenceLastInstanceLayerForward", getName().c_str()); - for (size_t seqId = 0; seqId < height; ++seqId) { + for (size_t seqId = 0; seqId < newBatchSize_; ++seqId) { int insId = config_.select_first() ? starts[seqId] : starts[seqId + 1] - 1; outputValue->subMatrix(seqId, 1, tmpDest_) ->assign(*(inputValue->subMatrix(insId, 1, tmpSrc_))); } - /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, - * thus, in this case, output_ has no sequenceStartPositions. - * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this - * case, we should compute the new sequenceStartPositions. - */ - if (type_) { - output_.degradeSequence(input, useGpu_); - } } if (biases_.get() != NULL) { @@ -138,23 +94,12 @@ void SequenceLastInstanceLayer::forward(PassType passType) { } void SequenceLastInstanceLayer::backward(const UpdateCallback& callback) { - /* activation, should set to 'linear' in most cases */ - backwardActivation(); - - if (biases_ && biases_->getWGrad()) { - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - - // Increasing the number of gradient - biases_->getParameterPtr()->incUpdate(callback); - } + SequencePoolLayer::backward(callback); MatrixPtr inputGrad = getInputGrad(0); MatrixPtr outputGrad = getOutputGrad(); - auto startPositions = - type_ ? getInput(0).subSequenceStartPositions->getVector(false) - : getInput(0).sequenceStartPositions->getVector(false); - const int* starts = startPositions->getData(); - size_t numSequences = startPositions->getSize() - 1; + const int* starts = startPositions_->getData(false); + size_t numSequences = startPositions_->getSize() - 1; if (inputGrad) { AsyncGpuBlock asyncGpuBlock; diff --git a/paddle/gserver/layers/SequencePoolLayer.cpp b/paddle/gserver/layers/SequencePoolLayer.cpp new file mode 100644 index 0000000000000..55be73d363df1 --- /dev/null +++ b/paddle/gserver/layers/SequencePoolLayer.cpp @@ -0,0 +1,84 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/utils/Logging.h" +#include "SequencePoolLayer.h" + +namespace paddle { + +bool SequencePoolLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + // seqlastins/max/average layer should have exactly 1 input + CHECK_EQ(1U, inputLayers_.size()); + + /* initialize biases_ */ + if (biasParameter_.get() != NULL) { + biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); + } + // transform to which sequence type + if (config_.trans_type() == "non-seq") { + type_ = kNonSeq; + } else if (config_.trans_type() == "seq") { + type_ = kSeq; + } else { + LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); + } + setNeedSequenceInfo(false); + return true; +} + +void SequencePoolLayer::forward(PassType passType) { + Layer::forward(passType); + + const Argument& input = getInput(0); + newBatchSize_ = type_ ? input.getNumSubSequences() : input.getNumSequences(); + size_t dim = getSize(); + // check + CHECK_EQ(dim, input.value->getWidth()); + startPositions_ = + type_ ? input.subSequenceStartPositions : input.sequenceStartPositions; + auto starts = startPositions_->getVector(false); + CHECK_EQ(starts->getData()[newBatchSize_], input.getBatchSize()); + CHECK_EQ(newBatchSize_, starts->getSize() - 1); + + resetOutput(newBatchSize_, dim); + if (type_) { + CHECK(input.subSequenceStartPositions) + << "when trans_type = seq, input must hasSubseq"; + } + /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, + * thus, in this case, output_ has no sequenceStartPositions. + * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this + * case, we should compute the new sequenceStartPositions. + */ + if (type_) { + output_.degradeSequence(input, useGpu_); + } +} + +void SequencePoolLayer::backward(const UpdateCallback& callback) { + /* Do derivation */ { backwardActivation(); } + + if (biases_ && biases_->getWGrad()) { + biases_->getWGrad()->collectBias(*getOutputGrad(), 1); + + // Increasing the number of gradient + biases_->getParameterPtr()->incUpdate(callback); + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/SequencePoolLayer.h b/paddle/gserver/layers/SequencePoolLayer.h new file mode 100644 index 0000000000000..669af80e1d447 --- /dev/null +++ b/paddle/gserver/layers/SequencePoolLayer.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "paddle/math/Matrix.h" + +namespace paddle { +/** + * A base layer for SequenceLastInstanceLayer/AverageLayer/MaxLayer. + * + * Input: one or more sequences. Each sequence contains some instances. + * If SequenceLevel = kNonSeq: + * Output: output size is the number of input sequences (NOT input instances) + * output[i] = seqlastin/average/max_{for each instance in this + * sequence}{input[i]} + * If SequenceLevel = kSeq: + * Check input sequence must has sub-sequence + * Output: output size is the number of input sub-sequences + * output[i] = seqlastin/average/max_{for each instance in this + * sub-sequence}{input[i]} + * + * The config file api is pooling_layer. + */ + +class SequencePoolLayer : public Layer { +protected: + int type_; + std::unique_ptr biases_; + enum SequenceLevel { kNonSeq = 0, kSeq = 1 }; + size_t newBatchSize_; + ICpuGpuVectorPtr startPositions_; + +public: + explicit SequencePoolLayer(const LayerConfig& config) : Layer(config) {} + + virtual ~SequencePoolLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + void backward(const UpdateCallback& callback = nullptr); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp new file mode 100644 index 0000000000000..2fcfc8e1ae68a --- /dev/null +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp @@ -0,0 +1,132 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "SpatialPyramidPoolLayer.h" + +namespace paddle { + +REGISTER_LAYER(spp, SpatialPyramidPoolLayer); + +ProjectionConfig SpatialPyramidPoolLayer::getConfig(size_t imgSizeW, + size_t imgSizeH, + size_t channels, + size_t pyramidLevel, + std::string& poolType) { + ProjectionConfig config; + config.set_type("pool"); + PoolConfig* conf = config.mutable_pool_conf(); + conf->set_channels(channels); + conf->set_img_size(imgSizeW); + conf->set_img_size_y(imgSizeH); + conf->set_pool_type(poolType); + + int numBins = std::pow(2, pyramidLevel); + + int sizeH = std::ceil(imgSizeH / static_cast(numBins)); + int paddingH = (sizeH * numBins - imgSizeH + 1) / 2; + int outSizeH = outputSize(imgSizeH, sizeH, paddingH, sizeH, true); + + int sizeW = std::ceil(imgSizeW / static_cast(numBins)); + int paddingW = (sizeW * numBins - imgSizeW + 1) / 2; + int outSizeW = outputSize(imgSizeW, sizeW, paddingW, sizeW, true); + + conf->set_stride(sizeW); + conf->set_stride_y(sizeH); + conf->set_size_x(sizeW); + conf->set_size_y(sizeH); + conf->set_padding(paddingW); + conf->set_padding_y(paddingH); + conf->set_output_x(outSizeW); + conf->set_output_y(outSizeH); + config.set_output_size(outSizeH * outSizeW * channels); + return config; +} + +size_t SpatialPyramidPoolLayer::getSize() { + CHECK_EQ(inputLayers_.size(), 1UL); + size_t layerSize = 0; + const SppConfig& sppConf = config_.inputs(0).spp_conf(); + imgSizeH_ = inputLayers_[0]->getOutput().getFrameHeight(); + imgSizeW_ = inputLayers_[0]->getOutput().getFrameWidth(); + if (imgSizeH_ == 0) { + imgSizeH_ = sppConf.has_img_size_y() ? sppConf.img_size_y() : imgSizeW_; + } + if (imgSizeW_ == 0) { + imgSizeW_ = sppConf.img_size(); + } + + size_t outputH = 1; + size_t outputW = (std::pow(4, pyramidHeight_) - 1) / (4 - 1); + + layerSize = outputH * outputW * channels_; + return layerSize; +} + +bool SpatialPyramidPoolLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + CHECK_EQ(config_.inputs_size(), 1); + + const SppConfig& sppConf = config_.inputs(0).spp_conf(); + pyramidHeight_ = sppConf.pyramid_height(); + poolType_ = sppConf.pool_type(); + + channels_ = sppConf.channels(); + imgSizeW_ = sppConf.img_size(); + imgSizeH_ = sppConf.has_img_size_y() ? sppConf.img_size_y() : imgSizeW_; + poolProjections_.reserve(pyramidHeight_); + projCol_.reserve(pyramidHeight_); + projOutput_.resize(pyramidHeight_); + + size_t startCol = 0; + size_t endCol = 0; + for (size_t i = 0; i < pyramidHeight_; i++) { + poolProjections_.emplace_back(PoolProjection::create( + getConfig(imgSizeW_, imgSizeH_, channels_, i, poolType_), nullptr, + useGpu_)); + endCol += poolProjections_[i]->getOutputSize(); + projCol_.push_back(std::make_pair(startCol, endCol)); + startCol = endCol; + } + CHECK_EQ(endCol, getSize()); + return true; +} + +void SpatialPyramidPoolLayer::forward(PassType passType) { + Layer::forward(passType); + + int batchSize = getInput(0).getBatchSize(); + resetOutput(batchSize, getSize()); + for (size_t i = 0; i < pyramidHeight_; i++) { + size_t startCol = projCol_[i].first; + size_t endCol = projCol_[i].second; + projOutput_[i].value = output_.value->subColMatrix(startCol, endCol); + if (output_.grad) { + projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); + } + } + for (size_t i = 0; i < pyramidHeight_; i++) { + poolProjections_[i]->forward(&getInput(0), &projOutput_[i], passType); + } +} + +void SpatialPyramidPoolLayer::backward(const UpdateCallback& callback) { + for (size_t i = 0; i < pyramidHeight_; i++) { + if (poolProjections_[i]) { + poolProjections_[i]->backward(callback); + } + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.h b/paddle/gserver/layers/SpatialPyramidPoolLayer.h new file mode 100644 index 0000000000000..e15b6d2f85c6f --- /dev/null +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "PoolProjection.h" +#include "paddle/math/MathUtils.h" +#include "paddle/utils/Logging.h" + +namespace paddle { +/** + * @brief A layer for spatial pyramid pooling on the input image by taking + * the max, average, etc. within regions, so that the result vector of + * different sized images are of the same size. + * + * The config file api is spp_layer. + */ + +class SpatialPyramidPoolLayer : public Layer { +protected: + size_t channels_; + size_t imgSizeW_; + size_t imgSizeH_; + size_t pyramidHeight_; + std::string poolType_; + + std::vector> poolProjections_; + std::vector projOutput_; + std::vector> projCol_; + +public: + explicit SpatialPyramidPoolLayer(const LayerConfig& config) : Layer(config) {} + + ~SpatialPyramidPoolLayer() {} + + virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + ProjectionConfig getConfig(size_t sizeX_, size_t sizeY_, size_t channels, + size_t pyamidLevel_, std::string& poolType_); + size_t getSize(); + + virtual void forward(PassType passType); + virtual void backward(const UpdateCallback& callback = nullptr); +}; +} // namespace paddle diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index ff2abf7697317..0651d0b4733ea 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -20,6 +20,21 @@ add_unittest_without_exec(test_LayerGrad add_test(NAME test_LayerGrad COMMAND test_LayerGrad) +add_unittest_without_exec(test_ActivationGrad + test_ActivationGrad.cpp + LayerGradUtil.cpp + TestUtil.cpp) +add_test(NAME test_ActivationGrad + COMMAND test_ActivationGrad) +################# test_ConvTrans ####################### +add_unittest_without_exec(test_ConvTrans + test_ConvTrans.cpp + LayerGradUtil.cpp + TestUtil.cpp) + +add_test(NAME test_ConvTrans + COMMAND test_ConvTrans) + ################## test_Evaluator ####################### add_unittest(test_Evaluator test_Evaluator.cpp diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index 552a6c5b41c7f..bc7bee0e4bbc8 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -669,12 +669,14 @@ void testLayerGrad(TestConfig testConf, string testLayerName, size_t batchSize, void testProjectionGrad(ProjectionConfig conf, InputType inputType, size_t parameterSize, size_t batchSize, bool useGpu, - bool testState) { + bool testState, int biasSize, bool sharedBias) { TestConfig config; conf.set_name(conf.type()); config.layerConfig.set_type("mixed"); config.layerConfig.set_size(conf.output_size()); - config.biasSize = config.layerConfig.size(); + config.biasSize = biasSize == 0 ? config.layerConfig.size() : biasSize; + config.layerConfig.set_bias_size(config.biasSize); + config.layerConfig.set_shared_biases(sharedBias); config.inputDefs.push_back( {inputType, "layer_0", conf.input_size(), parameterSize}); *config.layerConfig.add_inputs()->mutable_proj_conf() = conf; diff --git a/paddle/gserver/tests/LayerGradUtil.h b/paddle/gserver/tests/LayerGradUtil.h index 1e608dc0620ab..3b9ec803959b3 100644 --- a/paddle/gserver/tests/LayerGradUtil.h +++ b/paddle/gserver/tests/LayerGradUtil.h @@ -217,7 +217,8 @@ void testLayerGrad(TestConfig testConf, string testLayerName, size_t batchSize, void testProjectionGrad(ProjectionConfig conf, InputType inputType, size_t parameterSize, size_t batchSize, bool useGpu, - bool testState = false); + bool testState = false, int biasSize = 0, + bool sharedBias = false); void testOperatorGrad(TestConfig& config, OperatorConfig& operatorConf, size_t batchSize, bool useGpu, bool testState = false); diff --git a/paddle/gserver/tests/__init__.py b/paddle/gserver/tests/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/paddle/gserver/tests/__init__.py +++ b/paddle/gserver/tests/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/gserver/tests/img_conv_a.conf b/paddle/gserver/tests/img_conv_a.conf new file mode 100644 index 0000000000000..940589ed9ac24 --- /dev/null +++ b/paddle/gserver/tests/img_conv_a.conf @@ -0,0 +1,39 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +conv1 = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=16, stride=1, + bias_attr=False, + act=ReluActivation()) +conv2 = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=16, stride=1, + bias_attr=False, + act=ReluActivation()) + +concat = concat_layer(input=[conv1, conv2]) + +conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=16, stride=1, + bias_attr=True, + act=LinearActivation()) + +outputs(concat, conv) diff --git a/paddle/gserver/tests/img_conv_b.conf b/paddle/gserver/tests/img_conv_b.conf new file mode 100644 index 0000000000000..8ca9c94541504 --- /dev/null +++ b/paddle/gserver/tests/img_conv_b.conf @@ -0,0 +1,32 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +proj1 = conv_projection(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=16, stride=1) +proj2 = conv_projection(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=16, stride=1) +concat = concat_layer(input=[proj1, proj2], bias_attr=False, act=ReluActivation()) + +proj = conv_projection(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=16, stride=1) + +with mixed_layer(bias_attr=True, act=LinearActivation()) as conv: + conv += proj + +outputs(concat, conv) diff --git a/paddle/gserver/tests/img_pool_a.conf b/paddle/gserver/tests/img_pool_a.conf new file mode 100644 index 0000000000000..5938e7611201c --- /dev/null +++ b/paddle/gserver/tests/img_pool_a.conf @@ -0,0 +1,46 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=8,stride=1) +maxpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + img_width=16, + pool_type=MaxPooling(), +) +avgpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + img_width=16, + pool_type=AvgPooling(), +) + +outputs([maxpool, avgpool]) diff --git a/paddle/gserver/tests/img_pool_b.conf b/paddle/gserver/tests/img_pool_b.conf new file mode 100644 index 0000000000000..6ea9649b3f1ea --- /dev/null +++ b/paddle/gserver/tests/img_pool_b.conf @@ -0,0 +1,44 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=8, stride=1) +maxpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + pool_type=CudnnMaxPooling(), +) + +avgpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + pool_type=CudnnAvgPooling(), +) + +outputs([maxpool, avgpool]) diff --git a/paddle/gserver/tests/pyDataProvider.py b/paddle/gserver/tests/pyDataProvider.py index c3155e7adea04..91863b4175b1a 100644 --- a/paddle/gserver/tests/pyDataProvider.py +++ b/paddle/gserver/tests/pyDataProvider.py @@ -16,72 +16,79 @@ import struct import traceback + def header_creator(): ret = "" - ret += struct.pack('i', 3) # slot num - ret += struct.pack('i', 1) # sequence flag - ret += struct.pack('i', 0) # slot0 dense type - ret += struct.pack('i', 3) # slot0 dim - ret += struct.pack('i', 1) # slot1 sparse non value type - ret += struct.pack('i', 7) # slot1 dim - ret += struct.pack('i', 3) # slot2 index type - ret += struct.pack('i', 2) # slot2 dim + ret += struct.pack('i', 3) # slot num + ret += struct.pack('i', 1) # sequence flag + ret += struct.pack('i', 0) # slot0 dense type + ret += struct.pack('i', 3) # slot0 dim + ret += struct.pack('i', 1) # slot1 sparse non value type + ret += struct.pack('i', 7) # slot1 dim + ret += struct.pack('i', 3) # slot2 index type + ret += struct.pack('i', 2) # slot2 dim return ret + def dense_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot0 sample num - for i in range(sample_num): # slot0 value + ret += struct.pack('i', sample_num) # slot0 sample num + for i in range(sample_num): # slot0 value ret += struct.pack('f', 1.0) ret += struct.pack('f', 2.0) ret += struct.pack('f', 3.0) return ret + def sparse_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot1 sample num - for i in range(sample_num): # slot1 index + ret += struct.pack('i', sample_num) # slot1 sample num + for i in range(sample_num): # slot1 index ret += struct.pack('i', i * 2) - ret += struct.pack('i', sample_num * 2) #slot1 length - for i in range(sample_num): # slot1 value + ret += struct.pack('i', sample_num * 2) #slot1 length + for i in range(sample_num): # slot1 value ret += struct.pack('i', 1) ret += struct.pack('i', 2) return ret + def index_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot2 sample num - for i in range(sample_num): # slot2 value + ret += struct.pack('i', sample_num) # slot2 sample num + for i in range(sample_num): # slot2 value ret += struct.pack('i', 0) return ret + def sequenceStartPositions_creator(): ret = "" - ret += struct.pack('i', 2) # slot0 sequence num - ret += struct.pack('i', 0) # slot0 sequence value1 - ret += struct.pack('i', 1) # slot0 sequence value2 - ret += struct.pack('i', 1) # slot1 sequence num - ret += struct.pack('i', 0) # slot1 sequence value1 - ret += struct.pack('i', 2) # slot2 sequence num - ret += struct.pack('i', 0) # slot2 sequence value1 - ret += struct.pack('i', 1) # slot2 sequence value2 + ret += struct.pack('i', 2) # slot0 sequence num + ret += struct.pack('i', 0) # slot0 sequence value1 + ret += struct.pack('i', 1) # slot0 sequence value2 + ret += struct.pack('i', 1) # slot1 sequence num + ret += struct.pack('i', 0) # slot1 sequence value1 + ret += struct.pack('i', 2) # slot2 sequence num + ret += struct.pack('i', 0) # slot2 sequence value1 + ret += struct.pack('i', 1) # slot2 sequence value2 return ret + def subSequenceStartPositions_creator(): ret = "" - ret += struct.pack('i', 3) # slot0 subsequence num - ret += struct.pack('i', 0) # slot0 subsequence value1 - ret += struct.pack('i', 1) # slot0 subsequence value2 - ret += struct.pack('i', 2) # slot0 subsequence value3 - ret += struct.pack('i', 2) # slot1 subsequence num - ret += struct.pack('i', 0) # slot1 subsequence value1 - ret += struct.pack('i', 1) # slot1 subsequence value2 - ret += struct.pack('i', 3) # slot2 subsequence num - ret += struct.pack('i', 0) # slot2 subsequence value1 - ret += struct.pack('i', 1) # slot2 subsequence value2 - ret += struct.pack('i', 2) # slot2 subsequence value3 + ret += struct.pack('i', 3) # slot0 subsequence num + ret += struct.pack('i', 0) # slot0 subsequence value1 + ret += struct.pack('i', 1) # slot0 subsequence value2 + ret += struct.pack('i', 2) # slot0 subsequence value3 + ret += struct.pack('i', 2) # slot1 subsequence num + ret += struct.pack('i', 0) # slot1 subsequence value1 + ret += struct.pack('i', 1) # slot1 subsequence value2 + ret += struct.pack('i', 3) # slot2 subsequence num + ret += struct.pack('i', 0) # slot2 subsequence value1 + ret += struct.pack('i', 1) # slot2 subsequence value2 + ret += struct.pack('i', 2) # slot2 subsequence value3 return ret + class SimpleDataProvider: def __init__(self, *file_list): self.file_list = file_list @@ -93,17 +100,18 @@ def reset(self): pass def getHeader(self): - return header_creator() + return header_creator() def getNextBatch(self, batch_size): ret = "" - ret += struct.pack('i', 2) # batch size - ret += dense_value_creator(2) # slot0 - ret += sparse_value_creator(2) # slot1 - ret += index_value_creator(2) # slot2 + ret += struct.pack('i', 2) # batch size + ret += dense_value_creator(2) # slot0 + ret += sparse_value_creator(2) # slot1 + ret += index_value_creator(2) # slot2 ret += sequenceStartPositions_creator() return ret + class SimpleNestDataProvider: def __init__(self, *file_list): self.file_list = file_list @@ -119,14 +127,15 @@ def getHeader(self): def getNextBatch(self, batch_size): ret = "" - ret += struct.pack('i', 2) # batch size - ret += dense_value_creator(4) # slot0 - ret += sparse_value_creator(4) # slot1 - ret += index_value_creator(4) # slot2 + ret += struct.pack('i', 2) # batch size + ret += dense_value_creator(4) # slot0 + ret += sparse_value_creator(4) # slot1 + ret += index_value_creator(4) # slot2 ret += sequenceStartPositions_creator() ret += subSequenceStartPositions_creator() return ret + if __name__ == "__main__": # test code data_provider = SimpleDataProvider('./test_batch') diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py index 347d5891b906b..715ac08a42d05 100644 --- a/paddle/gserver/tests/rnn_data_provider.py +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -14,26 +14,84 @@ from paddle.trainer.PyDataProvider2 import * +# Note that each config should has an independent provider +# in current design of PyDataProvider2. +####################################################### data = [ [[[1, 3, 2], [4, 5, 2]], 0], [[[0, 2], [2, 5], [0, 1, 2]], 1], ] -@provider(input_types=[integer_value_sub_sequence(10), - integer_value(2)], - should_shuffle=False) +# Used for sequence_nest_rnn.conf +@provider( + input_types=[integer_value_sub_sequence(10), integer_value(3)], + should_shuffle=False) def process_subseq(settings, file_name): for d in data: yield d -@provider(input_types=[integer_value_sequence(10), - integer_value(2)], - should_shuffle=False) +# Used for sequence_rnn.conf +@provider( + input_types=[integer_value_sequence(10), integer_value(3)], + should_shuffle=False) def process_seq(settings, file_name): for d in data: seq = [] for subseq in d[0]: seq += subseq yield seq, d[1] + + +# Used for sequence_nest_rnn_multi_input.conf +@provider( + input_types=[integer_value_sub_sequence(10), integer_value(3)], + should_shuffle=False) +def process_subseq2(settings, file_name): + for d in data: + yield d + + +# Used for sequence_rnn_multi_input.conf +@provider( + input_types=[integer_value_sequence(10), integer_value(3)], + should_shuffle=False) +def process_seq2(settings, file_name): + for d in data: + seq = [] + for subseq in d[0]: + seq += subseq + yield seq, d[1] + + +########################################################### +data2 = [ + [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]], 0], + [[[0, 2], [2, 5], [0, 1, 2]], [[1, 5], [4], [2, 3, 6, 1]], 1], +] + + +# Used for sequence_nest_rnn_multi_unequalength_inputs.conf +@provider( + input_types=[ + integer_value_sub_sequence(10), integer_value_sub_sequence(10), + integer_value(2) + ], + should_shuffle=False) +def process_unequalength_subseq(settings, file_name): + for d in data2: + yield d + + +# Used for sequence_rnn_multi_unequalength_inputs.conf +@provider( + input_types=[ + integer_value_sequence(10), integer_value_sequence(10), integer_value(2) + ], + should_shuffle=False) +def process_unequalength_seq(settings, file_name): + for d in data2: + words1 = reduce(lambda x, y: x + y, d[0]) + words2 = reduce(lambda x, y: x + y, d[1]) + yield words1, words2, d[2] diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index cbed1f15fc415..fab876fd30da0 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -20,8 +20,9 @@ def hook(settings, dict_file, **kwargs): settings.word_dict = dict_file - settings.input_types = [integer_value_sequence(len(settings.word_dict)), - integer_value_sequence(3)] + settings.input_types = [ + integer_value_sequence(len(settings.word_dict)), integer_value(3) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -32,16 +33,19 @@ def process(settings, file_name): label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if - w in settings.word_dict] - yield word_slot, [label] + word_slot = [ + settings.word_dict[w] for w in words if w in settings.word_dict + ] + yield word_slot, label ## for hierarchical sequence network def hook2(settings, dict_file, **kwargs): settings.word_dict = dict_file - settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), - integer_value_sub_sequence(3)] + settings.input_types = [ + integer_value_sub_sequence(len(settings.word_dict)), + integer_value_sequence(3) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -55,9 +59,11 @@ def process2(settings, file_name): label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if - w in settings.word_dict] - label_list.append([label]) + word_slot = [ + settings.word_dict[w] for w in words + if w in settings.word_dict + ] + label_list.append(label) word_slot_list.append(word_slot) else: yield word_slot_list, label_list diff --git a/paddle/gserver/tests/sequence_layer_group.conf b/paddle/gserver/tests/sequence_layer_group.conf index ac031b31280df..087aa96ccb5a7 100644 --- a/paddle/gserver/tests/sequence_layer_group.conf +++ b/paddle/gserver/tests/sequence_layer_group.conf @@ -21,15 +21,16 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources2(train_list='gserver/tests/Sequence/train.list', - test_list=None, - module='sequenceGen', - obj='process', - args={"dict_file":dict_file}) +define_py_data_sources2( + train_list='gserver/tests/Sequence/train.list', + test_list=None, + module='sequenceGen', + obj='process', + args={"dict_file": dict_file}) settings(batch_size=5) ######################## network configure ################################ -dict_dim = len(open(dict_path,'r').readlines()) +dict_dim = len(open(dict_path, 'r').readlines()) word_dim = 128 hidden_dim = 256 label_dim = 3 @@ -39,21 +40,24 @@ data = data_layer(name="word", size=dict_dim) emb = embedding_layer(input=data, size=word_dim) # (lstm_input + lstm) is equal to lstmemory -with mixed_layer(size=hidden_dim*4) as lstm_input: +with mixed_layer(size=hidden_dim * 4) as lstm_input: lstm_input += full_matrix_projection(input=emb) -lstm = lstmemory_group(input=lstm_input, - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) +lstm = lstmemory_group( + input=lstm_input, + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) lstm_last = last_seq(input=lstm) -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: +with mixed_layer( + size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output: output += full_matrix_projection(input=lstm_last) -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +outputs( + classification_cost( + input=output, label=data_layer( + name="label", size=1))) diff --git a/paddle/gserver/tests/sequence_nest_layer_group.conf b/paddle/gserver/tests/sequence_nest_layer_group.conf index 38c60b657b969..93a0f6da7905c 100644 --- a/paddle/gserver/tests/sequence_nest_layer_group.conf +++ b/paddle/gserver/tests/sequence_nest_layer_group.conf @@ -21,15 +21,16 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources2(train_list='gserver/tests/Sequence/train.list.nest', - test_list=None, - module='sequenceGen', - obj='process2', - args={"dict_file":dict_file}) +define_py_data_sources2( + train_list='gserver/tests/Sequence/train.list.nest', + test_list=None, + module='sequenceGen', + obj='process2', + args={"dict_file": dict_file}) settings(batch_size=2) ######################## network configure ################################ -dict_dim = len(open(dict_path,'r').readlines()) +dict_dim = len(open(dict_path, 'r').readlines()) word_dim = 128 hidden_dim = 256 label_dim = 3 @@ -38,37 +39,46 @@ data = data_layer(name="word", size=dict_dim) emb_group = embedding_layer(input=data, size=word_dim) + # (lstm_input + lstm) is equal to lstmemory def lstm_group(lstm_group_input): - with mixed_layer(size=hidden_dim*4) as group_input: - group_input += full_matrix_projection(input=lstm_group_input) + with mixed_layer(size=hidden_dim * 4) as group_input: + group_input += full_matrix_projection(input=lstm_group_input) - lstm_output = lstmemory_group(input=group_input, - name="lstm_group", - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + lstm_output = lstmemory_group( + input=group_input, + name="lstm_group", + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) return lstm_output -lstm_nest_group = recurrent_group(input=SubsequenceInput(emb_group), - step=lstm_group, - name="lstm_nest_group") + +lstm_nest_group = recurrent_group( + input=SubsequenceInput(emb_group), step=lstm_group, name="lstm_nest_group") # hasSubseq ->(seqlastins) seq -lstm_last = last_seq(input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) +lstm_last = last_seq( + input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) # seq ->(expand) hasSubseq -lstm_expand = expand_layer(input=lstm_last, expand_as=emb_group, expand_level=ExpandLevel.FROM_SEQUENCE) +lstm_expand = expand_layer( + input=lstm_last, + expand_as=emb_group, + expand_level=ExpandLevel.FROM_SEQUENCE) # hasSubseq ->(average) seq -lstm_average = pooling_layer(input=lstm_expand, - pooling_type=AvgPooling(), - agg_level=AggregateLevel.EACH_SEQUENCE) +lstm_average = pooling_layer( + input=lstm_expand, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: +with mixed_layer( + size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output: output += full_matrix_projection(input=lstm_average) -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +outputs( + classification_cost( + input=output, label=data_layer( + name="label", size=1))) diff --git a/paddle/gserver/tests/sequence_nest_rnn.conf b/paddle/gserver/tests/sequence_nest_rnn.conf index 62b8c5d072d7b..93b08eb2f8746 100644 --- a/paddle/gserver/tests/sequence_nest_rnn.conf +++ b/paddle/gserver/tests/sequence_nest_rnn.conf @@ -56,9 +56,8 @@ def outer_step(x): last = last_seq(input=inner_rnn_output, name="outer_rnn_state") # "return last" should also work. But currently RecurrentGradientMachine - # does not handle it correctly. Current implementation requires that - # all the out links are from sequences. However, it does not report error - # when the out links are not sequences. + # does not handle it, and will report error: In hierachical RNN, all out + # links should be from sequences now. return inner_rnn_output out = recurrent_group( diff --git a/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf index e01b3f8e7aa5c..0614958b4719d 100644 --- a/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf +++ b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf @@ -19,7 +19,7 @@ from paddle.trainer_config_helpers import * define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', test_list=None, module='rnn_data_provider', - obj='process_subseq') + obj='process_subseq2') settings(batch_size=2, learning_rate=0.01) @@ -57,9 +57,8 @@ def outer_step(wid, x): last = last_seq(input=inner_rnn_output, name="outer_rnn_state") # "return last" should also work. But currently RecurrentGradientMachine - # does not handle it correctly. Current implementation requires that - # all the out links are from sequences. However, it does not report error - # when the out links are not sequences. + # does not handle it, and will report error: In hierachical RNN, all out + # links should be from sequences now. return inner_rnn_output out = recurrent_group( diff --git a/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf b/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf new file mode 100644 index 0000000000000..d0b9450f4b9f9 --- /dev/null +++ b/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf @@ -0,0 +1,106 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_unequalength_subseq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 2 + +speaker1 = data_layer(name="word1", size=dict_dim) +speaker2 = data_layer(name="word2", size=dict_dim) + +emb1 = embedding_layer(input=speaker1, size=word_dim) +emb2 = embedding_layer(input=speaker2, size=word_dim) + +# This hierachical RNN is designed to be equivalent to the simple RNN in +# sequence_rnn_multi_unequalength_inputs.conf + +def outer_step(x1, x2): + outer_mem1 = memory(name = "outer_rnn_state1", size = hidden_dim) + outer_mem2 = memory(name = "outer_rnn_state2", size = hidden_dim) + def inner_step1(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem1) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + def inner_step2(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem2) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + encoder1 = recurrent_group( + step = inner_step1, + name = 'inner1', + input = x1) + + encoder2 = recurrent_group( + step = inner_step2, + name = 'inner2', + input = x2) + + sentence_last_state1 = last_seq(input = encoder1, name = 'outer_rnn_state1') + sentence_last_state2_ = last_seq(input = encoder2, name = 'outer_rnn_state2') + + encoder1_expand = expand_layer(input = sentence_last_state1, + expand_as = encoder2) + + return [encoder1_expand, encoder2] + + +encoder1_rep, encoder2_rep = recurrent_group( + name="outer", + step=outer_step, + input=[SubsequenceInput(emb1), SubsequenceInput(emb2)], + targetInlink=emb2) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) + +rep = last_seq(input=context) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) + diff --git a/paddle/gserver/tests/sequence_rnn_multi_input.conf b/paddle/gserver/tests/sequence_rnn_multi_input.conf index 968621cab59be..51881e21d971b 100644 --- a/paddle/gserver/tests/sequence_rnn_multi_input.conf +++ b/paddle/gserver/tests/sequence_rnn_multi_input.conf @@ -19,7 +19,7 @@ from paddle.trainer_config_helpers import * define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', test_list=None, module='rnn_data_provider', - obj='process_seq') + obj='process_seq2') settings(batch_size=2, learning_rate=0.01) diff --git a/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.conf b/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.conf new file mode 100644 index 0000000000000..28b1cb98cf132 --- /dev/null +++ b/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.conf @@ -0,0 +1,75 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_unequalength_seq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 2 + +speaker1 = data_layer(name="word1", size=dict_dim) +speaker2 = data_layer(name="word2", size=dict_dim) + +emb1 = embedding_layer(input=speaker1, size=word_dim) +emb2 = embedding_layer(input=speaker2, size=word_dim) + +# This hierachical RNN is designed to be equivalent to the RNN in +# sequence_nest_rnn_multi_unequalength_inputs.conf + +def step(x1, x2): + def calrnn(y): + mem = memory(name = 'rnn_state_' + y.name, size = hidden_dim) + out = fc_layer(input = [y, mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'rnn_state_' + y.name) + return out + + encoder1 = calrnn(x1) + encoder2 = calrnn(x2) + return [encoder1, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="stepout", + step=step, + input=[emb1, emb2]) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) + +rep = last_seq(input=context) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) + diff --git a/paddle/gserver/tests/test_ActivationGrad.cpp b/paddle/gserver/tests/test_ActivationGrad.cpp new file mode 100644 index 0000000000000..2c5d17090dfc7 --- /dev/null +++ b/paddle/gserver/tests/test_ActivationGrad.cpp @@ -0,0 +1,66 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/gserver/layers/DataLayer.h" +#include "ModelConfig.pb.h" +#include "paddle/trainer/Trainer.h" + +#include "TestUtil.h" +#include "LayerGradUtil.h" + +using namespace paddle; // NOLINT +using namespace std; // NOLINT + +P_DECLARE_bool(use_gpu); +P_DECLARE_bool(thread_local_rand_use_global_seed); + +void testActivation(const string& act) { + LOG(INFO) << "test activation: " << act; + size_t size = 10; + TestConfig config; + config.biasSize = 0; + config.layerConfig.set_type("addto"); + config.layerConfig.set_size(size); + config.layerConfig.set_active_type(act); + config.inputDefs.push_back({INPUT_DATA, "layer_0", size, 0}); + config.layerConfig.add_inputs(); + for (auto useGpu : {false, true}) { + testLayerGrad(config, + act + "_activation", + 100, + /* trans= */false, + useGpu, + /* useWeight */true); + } +} + +TEST(Activation, activation) { + auto types = ActivationFunction::getAllRegisteredTypes(); + std::set excluded{"sequence_softmax"}; + for (auto type : types) { + if (excluded.count(type)) continue; + testActivation(type); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + FLAGS_thread_local_rand_use_global_seed = true; + srand(1); + return RUN_ALL_TESTS(); +} diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp new file mode 100644 index 0000000000000..bff7222b29907 --- /dev/null +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -0,0 +1,246 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/gserver/layers/DataLayer.h" +#include "ModelConfig.pb.h" +#include "paddle/trainer/Trainer.h" +#include "paddle/utils/GlobalConstants.h" +#include "paddle/gserver/layers/ExpandConvTransLayer.h" +#include "paddle/math/MathUtils.h" + +#include "TestUtil.h" +#include "LayerGradUtil.h" + +using namespace paddle; // NOLINT +using namespace std; // NOLINT + +P_DECLARE_bool(use_gpu); +P_DECLARE_int32(gpu_id); +P_DECLARE_double(checkgrad_eps); +P_DECLARE_bool(thread_local_rand_use_global_seed); +P_DECLARE_bool(prev_batch_state); + +// Test that the convTrans forward is the same as conv backward +TEST(Layer, convTransLayerFwd) { + // Setting up conv-trans layer + TestConfig configt; + configt.biasSize = 3; + configt.layerConfig.set_type("exconvt"); + configt.layerConfig.set_num_filters(3); + configt.layerConfig.set_partial_sum(1); + configt.layerConfig.set_shared_biases(true); + + configt.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 384}); + LayerInputConfig* input = configt.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(4); + conv->set_channels(16); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(3 / conv->groups()); + conv->set_img_size(16); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); + configt.layerConfig.set_size(conv->img_size() * conv->img_size() * + configt.layerConfig.num_filters()); + configt.layerConfig.set_name("convTrans"); + + // data layer initialize + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer(configt, &dataLayers, &datas, &layerMap, "convTrans", + 100, false, false); + // test layer initialize + std::vector parameters; + LayerPtr convtLayer; + initTestLayer(configt, &layerMap, ¶meters, &convtLayer); + convtLayer->getBiasParameter()->zeroMem(); + convtLayer->forward(PASS_GC); + + // Setting up conv-layer config + TestConfig config; + config.biasSize = 16; + config.layerConfig.set_type("exconv"); + config.layerConfig.set_num_filters(16); + config.layerConfig.set_partial_sum(1); + config.layerConfig.set_shared_biases(true); + + config.inputDefs.push_back({INPUT_DATA, "layer_1", 768, 384}); + input = config.layerConfig.add_inputs(); + conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(4); + conv->set_channels(3); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(conv->channels() / conv->groups()); + conv->set_img_size(16); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); + config.layerConfig.set_size(conv->output_x() * conv->output_x() * + config.layerConfig.num_filters()); + config.layerConfig.set_name("conv"); + + // data layer initialize + std::vector dataLayers2; + LayerMap layerMap2; + vector datas2; + initDataLayer(config, &dataLayers2, &datas2, &layerMap2, "conv", + 100, false, false); + // test layer initialize + std::vector parameters2; + LayerPtr convLayer; + initTestLayer(config, &layerMap2, ¶meters2, &convLayer); + + // Sync convLayer and convtLayer parameter + convLayer->getBiasParameter()->zeroMem(); + convLayer->getParameters()[0]->getBuf(PARAMETER_VALUE)->copyFrom( + *(convtLayer->getParameters()[0]->getBuf(PARAMETER_VALUE))); + + // Set convLayer outputGrad as convTransLayer input value + convLayer->forward(PASS_GC); + convLayer->getOutput().grad->copyFrom(*(dataLayers[0]->getOutputValue())); + + vector callbackFlags(parameters2.size(), 0); + auto callback = [&](Parameter* para) { ++callbackFlags[para->getID()]; }; + convLayer->backward(callback); + + // Check that the convLayer backward is the same as convTransLayer forward + checkMatrixEqual(convtLayer->getOutputValue(), + dataLayers2[0]->getOutputGrad()); +} + + +// Do one forward pass of convTrans layer and check to see if its output +// matches the given result +void doOneConvtTest(size_t imgSize, size_t output_x, size_t stride, + size_t padding, size_t filter_size, MatrixPtr& result) { + TestConfig configt; + configt.biasSize = 1; + configt.layerConfig.set_type("exconvt"); + configt.layerConfig.set_num_filters(1); + configt.layerConfig.set_partial_sum(1); + configt.layerConfig.set_shared_biases(true); + + configt.inputDefs.push_back({INPUT_DATA, "layer_0", output_x * output_x, + filter_size * filter_size}); + LayerInputConfig* input = configt.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(filter_size); + conv->set_filter_size_y(filter_size); + conv->set_channels(1); + conv->set_padding(padding); + conv->set_padding_y(padding); + conv->set_stride(stride); + conv->set_stride_y(stride); + conv->set_groups(1); + conv->set_filter_channels(1); + conv->set_img_size(imgSize); + conv->set_output_x(output_x); + + configt.layerConfig.set_size(conv->img_size() * conv->img_size() * + configt.layerConfig.num_filters()); + configt.layerConfig.set_name("convTrans"); + + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer(configt, &dataLayers, &datas, &layerMap, "convTrans", + 1, false, false); + dataLayers[0]->getOutputValue()->zeroMem(); + dataLayers[0]->getOutputValue()->add(1.0); + + // test layer initialize + std::vector parameters; + LayerPtr convtLayer; + initTestLayer(configt, &layerMap, ¶meters, &convtLayer); + convtLayer->getBiasParameter()->zeroMem(); + convtLayer->getParameters()[0]->zeroMem(); + convtLayer->getParameters()[0]->getBuf(PARAMETER_VALUE)->add(1.0); + convtLayer->forward(PASS_GC); + + checkMatrixEqual(convtLayer->getOutputValue(), result); +} + +TEST(Layer, convTransLayerFwd2) { + MatrixPtr result; + result = Matrix::create(1, 5 * 5, false, false); + result->zeroMem(); + result->add(1.0); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 1, + /* stride */ 1, + /* padding */ 0, + /* filter_size */ 5, + result); + + float resultData[] = {1, 2, 2, 2, 1, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 1, 2, 2, 2, 1}; + result->setData(resultData); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 1, + /* padding */ 0, + /* filter_size */ 4, + result); + + float resultData2[] = {1, 2, 2, 2, 1, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 1, 2, 2, 2, 1}; + result->setData(resultData2); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 2, + /* padding */ 1, + /* filter_size */ 5, + result); + + float resultData3[] = {1, 1, 2, 1, 1, + 1, 1, 2, 1, 1, + 2, 2, 4, 2, 2, + 1, 1, 2, 1, 1, + 1, 1, 2, 1, 1}; + result->setData(resultData3); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 2, + /* padding */ 0, + /* filter_size */ 3, + result);} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + FLAGS_thread_local_rand_use_global_seed = true; + srand(1); + return RUN_ALL_TESTS(); +} diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 3150c31e4900c..a79dfe39c9bb2 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -13,14 +13,15 @@ See the License for the specific language governing permissions and limitations under the License. */ #include -#include #include -#include "paddle/gserver/layers/DataLayer.h" +#include #include "ModelConfig.pb.h" +#include "paddle/gserver/layers/DataLayer.h" #include "paddle/trainer/Trainer.h" +#include "paddle/math/MathUtils.h" -#include "TestUtil.h" #include "LayerGradUtil.h" +#include "TestUtil.h" using namespace paddle; // NOLINT using namespace std; // NOLINT @@ -134,6 +135,78 @@ TEST(Projection, identity) { } } +TEST(Projection, scaling) { + ProjectionConfig conf; + conf.set_type("scaling"); + conf.set_input_size(10); + conf.set_output_size(10); + for (auto useGpu : {false}) { + testProjectionGrad(conf, INPUT_DATA, /* parameterSize */ 1, + /* batchSize */ 100, useGpu); + } +} + +#ifndef PADDLE_ONLY_CPU +TEST(Projection, conv) { + const int NUM_FILTERS = 16; + const int FILTER_SIZE = 2; + const int FILTER_SIZE_Y = 3; + const int CHANNELS = 3; + const int IMAGE_SIZE = 16; + + ProjectionConfig conf; + conf.set_type("conv"); + conf.set_num_filters(NUM_FILTERS); + + ConvConfig* conv = conf.mutable_conv_conf(); + conv->set_filter_size(FILTER_SIZE); + conv->set_filter_size_y(FILTER_SIZE_Y); + conv->set_channels(CHANNELS); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(conv->channels() / conv->groups()); + conv->set_img_size(IMAGE_SIZE); + int output_x = + outputSize(conv->img_size(), conv->filter_size(), conv->padding(), + conv->stride(), /* caffeMode */ true); + int output_y = + outputSize(conv->img_size(), conv->filter_size_y(), conv->padding_y(), + conv->stride_y(), /* caffeMode */ true); + conv->set_output_x(output_x); + conf.set_input_size(IMAGE_SIZE * IMAGE_SIZE * CHANNELS); + conf.set_output_size(output_x * output_y * NUM_FILTERS); + + testProjectionGrad( + conf, INPUT_DATA, + /* parameterSize */ NUM_FILTERS * CHANNELS * FILTER_SIZE * FILTER_SIZE_Y, + /* batchSize */ 100, true, false, NUM_FILTERS, true); +} +#endif + +TEST(Layer, BilinearInterpLayer) { + TestConfig config; + config.layerConfig.set_type("bilinear_interp"); + config.biasSize = 0; + config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); + + LayerInputConfig* input = config.layerConfig.add_inputs(); + BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); + bilinear->set_img_size_x(32); + bilinear->set_img_size_y(32); + bilinear->set_num_channels(4); + + for (auto useGpu : {false, true}) { + for (auto outSize : {32, 64}) { + bilinear->set_out_size_x(outSize); + bilinear->set_out_size_y(outSize); + testLayerGrad(config, "bilinear_interp", 10, false, useGpu); + } + } +} + TEST(Layer, concat) { TestConfig config; config.biasSize = 0; @@ -254,14 +327,15 @@ void testConvLayer(const string& type, bool trans, bool useGpu) { conv->set_groups(1); conv->set_filter_channels(conv->channels() / conv->groups()); conv->set_img_size(16); - conv->set_output_x( - (2 * conv->padding() + conv->img_size() - conv->filter_size()) / - ((float)conv->stride()) + - 1.5); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); config.layerConfig.set_size(conv->output_x() * conv->output_x() * config.layerConfig.num_filters()); testLayerGrad(config, "conv", 100, trans, useGpu); + // Use small batch_size and useWeight=true to test biasGrad + testLayerGrad(config, "conv", 2, trans, useGpu, true, 0.02); } TEST(Layer, convLayer) { @@ -272,6 +346,46 @@ TEST(Layer, convLayer) { #endif } + +void testConvTransLayer(const string& type, bool trans, bool useGpu) { + TestConfig config; + config.biasSize = 3; + config.layerConfig.set_type(type); + config.layerConfig.set_num_filters(3); + config.layerConfig.set_partial_sum(1); + config.layerConfig.set_shared_biases(true); + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 288}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(3); + conv->set_channels(16); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(3 / conv->groups()); + conv->set_img_size(16); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); + + config.layerConfig.set_size(conv->img_size() * conv->img_size() * + config.layerConfig.num_filters()); + + testLayerGrad(config, "convTrans", 100, trans, useGpu); + // Use small batch_size and useWeight=true to test biasGrad + testLayerGrad(config, "convTrans", 2, trans, useGpu, true, 0.02); +} + +TEST(Layer, convTransLayer) { + for (auto useGpu : {false, true}) { + testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ useGpu); + } +} + TEST(Layer, blockExpandLayer) { TestConfig config; config.biasSize = 0; @@ -290,15 +404,13 @@ TEST(Layer, blockExpandLayer) { blockExpand->set_stride_x(2); blockExpand->set_stride_y(2); blockExpand->set_output_x( - 1 + - (2 * blockExpand->padding_x() + blockExpand->img_size_x() - - blockExpand->block_x() + blockExpand->stride_x() - 1) / - blockExpand->stride_x()); + outputSize(blockExpand->img_size_x(), blockExpand->block_x(), + blockExpand->padding_x(), blockExpand->stride_x(), + /* caffeMode */ false)); blockExpand->set_output_y( - 1 + - (2 * blockExpand->padding_y() + blockExpand->img_size_y() - - blockExpand->block_y() + blockExpand->stride_y() - 1) / - blockExpand->stride_y()); + outputSize(blockExpand->img_size_y(), blockExpand->block_y(), + blockExpand->padding_y(), blockExpand->stride_y(), + /* caffeMode */ false)); config.layerConfig.set_size(blockExpand->block_x() * blockExpand->block_y() * blockExpand->channels()); @@ -307,6 +419,24 @@ TEST(Layer, blockExpandLayer) { } } +TEST(Layer, maxoutLayer) { + TestConfig config; + config.biasSize = 0; + config.layerConfig.set_type("maxout"); + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + MaxOutConfig* maxout = input->mutable_maxout_conf(); + + maxout->set_img_size_x(32); + maxout->set_img_size_y(32); + maxout->set_channels(4); + maxout->set_groups(2); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "maxout", 10, false, useGpu); + } +} void testFcLayer(string format, size_t nnz) { TestConfig config; config.biasSize = 4096; @@ -409,7 +539,7 @@ TEST(Layer, multi_cross) { } } -TEST(Layer, multi_binary_label) { +TEST(Layer, multi_binary_label_sparse_mat) { TestConfig config; config.layerConfig.set_type("multi_binary_label_cross_entropy"); config.biasSize = 0; @@ -419,9 +549,26 @@ TEST(Layer, multi_binary_label) { config.layerConfig.add_inputs(); config.layerConfig.add_inputs(); - // Not support GPU now - testLayerGrad(config, "multi_binary_label_cross_entropy", 100, - /* trans */ false, /* useGpu */ false); + for (auto useGpu : {false, true}) { + testLayerGrad(config, "multi_binary_label_cross_entropy", 100, + /* trans */ false, useGpu); + } +} + +TEST(layer, multi_binary_label_id) { + TestConfig config; + config.layerConfig.set_type("multi_binary_label_cross_entropy"); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); + config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); + config.layerConfig.add_inputs(); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "multi_binary_label_cross_entropy", 100, + /* trans */ false, useGpu); + } } TEST(Layer, multi_cross_with_selfnorm) { @@ -791,21 +938,24 @@ void setPoolConfig(TestConfig* config, PoolConfig* pool, (*config).biasSize = 0; (*config).layerConfig.set_type("pool"); (*config).layerConfig.set_num_filters(16); - (*config).layerConfig.set_partial_sum(1); - (*config).layerConfig.set_shared_biases(true); + int kw = 3, kh = 3; + int pw = 0, ph = 0; + int sw = 2, sh = 2; pool->set_pool_type(poolType); pool->set_channels(16); - pool->set_size_x(3); - if (poolType == "cudnn-max-pool" || poolType == "cudnn-avg-pool") { - pool->set_padding(0); - } else { - pool->set_start(0); - } - pool->set_stride(2); - pool->set_output_x((pool->img_size() - pool->start() - pool->size_x()) / - ((float)pool->stride()) + - 1.5); + pool->set_size_x(kw); + pool->set_size_y(kh); + pool->set_start(0); + pool->set_padding(pw); + pool->set_padding_y(ph); + pool->set_stride(sw); + pool->set_stride_y(sh); + + int ow = outputSize(pool->img_size(), kw, pw, sw, /* caffeMode */ false); + int oh = outputSize(pool->img_size_y(), kh, ph, sh, /* caffeMode */ false); + pool->set_output_x(ow); + pool->set_output_y(oh); } void testPoolLayer(const string& poolType, bool trans, bool useGpu) { @@ -814,9 +964,10 @@ void testPoolLayer(const string& poolType, bool trans, bool useGpu) { LayerInputConfig* input = config.layerConfig.add_inputs(); PoolConfig* pool = input->mutable_pool_conf(); - setPoolConfig(&config, pool, poolType); pool->set_img_size(14); - config.layerConfig.set_size(pool->output_x() * pool->output_x() * + pool->set_img_size_y(14); + setPoolConfig(&config, pool, poolType); + config.layerConfig.set_size(pool->output_x() * pool->output_y() * pool->channels()); testLayerGrad(config, "pool", 100, trans, useGpu); @@ -829,11 +980,11 @@ void testPoolLayer2(const string& poolType, bool trans, bool useGpu) { LayerInputConfig* input = config.layerConfig.add_inputs(); PoolConfig* pool = input->mutable_pool_conf(); - setPoolConfig(&config, pool, poolType); pool->set_size_y(4); pool->set_stride_y(3); pool->set_img_size(10); pool->set_img_size_y(20); + setPoolConfig(&config, pool, poolType); pool->set_output_y((pool->img_size_y() - pool->start() - pool->size_y()) / ((float)pool->stride_y()) + 1.5); @@ -858,6 +1009,32 @@ TEST(Layer, PoolLayer) { #endif } +void testSppLayer(const string& poolType, const int pyramidHeight, bool trans, + bool useGpu) { + TestConfig config; + config.layerConfig.set_type("spp"); + config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + SppConfig* sppConfig = input->mutable_spp_conf(); + sppConfig->set_pool_type(poolType); + sppConfig->set_pyramid_height(pyramidHeight); + sppConfig->set_channels(16); + sppConfig->set_img_size(10); + sppConfig->set_img_size_y(20); + int outputSize = (std::pow(4, sppConfig->pyramid_height()) - 1) / (4 - 1); + config.layerConfig.set_size(outputSize * sppConfig->channels()); + testLayerGrad(config, "spp", 100, trans, useGpu); +} + +TEST(Layer, SpatialPyramidPoolLayer) { + for (auto useGpu : {false, true}) { + for (auto pyramidHeight : {1, 2, 3}) { + testSppLayer("avg-projection", pyramidHeight, false, useGpu); + testSppLayer("max-projection", pyramidHeight, false, useGpu); + } + } +} + TEST(Layer, rankCostLayer) { TestConfig config; config.layerConfig.set_type("rank-cost"); @@ -875,6 +1052,19 @@ TEST(Layer, rankCostLayer) { } } +TEST(Layer, sumCostLayer) { + TestConfig config; + config.layerConfig.set_type("sum_cost"); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "sum_cost", 100, false, useGpu); + } +} + TEST(Layer, weightedRankCostLayer) { TestConfig config; config.layerConfig.set_type("rank-cost"); @@ -935,7 +1125,7 @@ TEST(Layer, LstmLayer) { TestConfig config; config.layerConfig.set_type("lstmemory"); config.layerConfig.set_size(4); - config.layerConfig.set_active_type("sigmoid"); + config.layerConfig.set_active_type("tanh"); config.layerConfig.set_active_state_type("sigmoid"); config.layerConfig.set_active_gate_type("sigmoid"); config.biasSize = 28; @@ -1194,12 +1384,11 @@ TEST(Operator, conv) { conv->set_groups(1); conv->set_filter_channels(conv->channels() / conv->groups()); conv->set_img_size(IMAGE_SIZE); - int outputSize = - int(1.0 * (2 * conv->padding() + conv->img_size() - conv->filter_size()) / - conv->stride()) + - 1; - conv->set_output_x(outputSize); - config.layerConfig.set_size(outputSize * outputSize * + int output_x = + outputSize(conv->img_size(), conv->filter_size(), conv->padding(), + conv->stride(), /* caffeMode */ true); + conv->set_output_x(output_x); + config.layerConfig.set_size(output_x * output_x * config.layerConfig.num_filters()); config.layerConfig.set_size(conv->output_x() * conv->output_x() * NUM_FILTERS); @@ -1252,8 +1441,6 @@ TEST(Layer, MultiplexLayer) { } } - - int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index 1c6a8b0017fc9..8d3eac5aca8d1 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -116,6 +116,8 @@ void calcGradient(DataIn& in, DataOut& out, const std::string& configPath) { gradientMachine->start(trainer.getConfig(), nullptr); gradientMachine->forward(in.inArgs, &outArgs, PASS_TRAIN); for (size_t i = 0; i < in.outGrads.size(); i++) { + // If the all the layers in the config have no parameters, also + // not set NeedGradient(), the outArgs[i] will be nullptr. outArgs[i].grad->copyFrom(*in.outGrads[i]); } gradientMachine->backward(); @@ -225,6 +227,27 @@ TEST(Compare, concat_table) { compareNetwork(config_file_a, config_file_b); } +#ifndef PADDLE_ONLY_CPU +TEST(Compare, img_pool) { + std::string config_file_a = "./gserver/tests/img_pool_a.conf"; + std::string config_file_b = "./gserver/tests/img_pool_b.conf"; + bool useGpu = FLAGS_use_gpu; + FLAGS_use_gpu = true; + compareNetwork(config_file_a, config_file_b); + FLAGS_use_gpu = useGpu; +} + +TEST(Compare, img_conv) { + std::string config_file_a = "./gserver/tests/img_conv_a.conf"; + std::string config_file_b = "./gserver/tests/img_conv_b.conf"; + bool useGpu = FLAGS_use_gpu; + FLAGS_use_gpu = true; + compareNetwork(config_file_a, config_file_b); + FLAGS_use_gpu = useGpu; +} +#endif + + P_DEFINE_string(config_file_a, "", "config of one network to compare"); P_DEFINE_string(config_file_b, "", "config of another network to compare"); TEST(Compare, network) { diff --git a/paddle/gserver/tests/test_PyDataProvider2.cpp b/paddle/gserver/tests/test_PyDataProvider2.cpp index e75e53ab7f431..b9867a728d9b4 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.cpp +++ b/paddle/gserver/tests/test_PyDataProvider2.cpp @@ -117,7 +117,7 @@ TEST(PyDataProvider2, index_no_seq) { } TEST(PyDataProvider2, init_hook) { - paddle::PyObjectPtr pickle(PyImport_ImportModule("pickle")); + paddle::PyObjectPtr pickle = paddle::py::import("pickle"); paddle::PyObjectPtr globals( PyModule_GetDict(PyImport_AddModule("__main__"))); PyDict_SetItemString(globals.get(), "pickle", pickle.get()); @@ -353,6 +353,23 @@ TEST(PyDataProvider2, test_check) { } } +TEST(PyDataProvider2, multiThread) { + paddle::DataConfig config; + config.set_type("py2"); + config.set_files(FLAGS_train_list.c_str()); + config.set_load_data_module("test_PyDataProvider2"); + config.set_load_data_object("test_dense_no_seq"); + config.set_async_load_data(true); + + std::unique_ptr provider( + paddle::DataProvider::create(config, false)); + provider->reset(); + paddle::DataBatch batch; + provider->getNextBatch(100, &batch); + provider->reset(); + provider.reset(); +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); paddle::initMain(argc, argv); diff --git a/paddle/gserver/tests/test_PyDataProvider2.py b/paddle/gserver/tests/test_PyDataProvider2.py index 145fe85cff7d8..7ca30198fb1d0 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.py +++ b/paddle/gserver/tests/test_PyDataProvider2.py @@ -33,16 +33,19 @@ def test_init_hooker(setting, value, **kwargs): setting.value = value -@provider(input_types=[dense_vector(20, seq_type=SequenceType.NO_SEQUENCE)], - init_hook=test_init_hooker) +@provider( + input_types=[dense_vector( + 20, seq_type=SequenceType.NO_SEQUENCE)], + init_hook=test_init_hooker) def test_init_hook(setting, filename): for i in xrange(200): yield setting.value -@provider( - input_types=[ - sparse_binary_vector(30000, seq_type=SequenceType.NO_SEQUENCE)]) +@provider(input_types=[ + sparse_binary_vector( + 30000, seq_type=SequenceType.NO_SEQUENCE) +]) def test_sparse_non_value_no_seq(setting, filename): for i in xrange(200): yield [(i + 1) * (j + 1) for j in xrange(10)] @@ -77,28 +80,28 @@ def test_min_pool_size(setting, filename): yield random.randint(0, 100 - 1) -@provider(input_types=[index_slot(100, seq_type=SequenceType.SEQUENCE)], - can_over_batch_size=False, - calc_batch_size=lambda x: len(x[0])) +@provider( + input_types=[index_slot( + 100, seq_type=SequenceType.SEQUENCE)], + can_over_batch_size=False, + calc_batch_size=lambda x: len(x[0])) def test_can_over_batch_size(setting, filename): for _ in xrange(1 << 10): seq_len = random.randint(0, 99) yield [random.randint(0, 100 - 1) for _ in xrange(seq_len)] -@provider(input_types=[index_slot(10), index_slot(10)]) +@provider(input_types={'input1': index_slot(10), 'input2': index_slot(10)}) def test_input_order(setting, filename): for _ in xrange(1000): - yield { - 'input1': 0, - 'input2': 1 - } + yield {'input1': 0, 'input2': 1} -@provider(input_types=[index_slot(10)], - check=True, - check_fail_continue=True, - should_shuffle="123") # also test should shuffle +@provider( + input_types=[index_slot(10)], + check=True, + check_fail_continue=True, + should_shuffle="123") # also test should shuffle def test_check(settings, filename): yield_good_value = False @@ -108,4 +111,3 @@ def test_check(settings, filename): if i < 10: yield_good_value = True yield i - diff --git a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp index 550df0a31844e..d104db3e5b32d 100644 --- a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp +++ b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include #include #include @@ -24,7 +23,7 @@ limitations under the License. */ P_DECLARE_int32(seed); using namespace paddle; // NOLINT -using namespace std; // NOLINT +using namespace std; // NOLINT class TrainerForTest : public paddle::Trainer { public: void startTrain() { @@ -44,11 +43,10 @@ class TrainerForTest : public paddle::Trainer { */ size_t getTotalParameterSize() const { auto p = const_cast(this); - auto & params = p->getGradientMachine()->getParameters(); - return std::accumulate(params.begin(), params.end(), 0UL, - [](size_t a, const ParameterPtr& p){ - return a+p->getSize(); - }); + auto& params = p->getGradientMachine()->getParameters(); + return std::accumulate( + params.begin(), params.end(), 0UL, + [](size_t a, const ParameterPtr& p) { return a + p->getSize(); }); } }; @@ -73,6 +71,7 @@ void CalCost(const string& conf, const string& dir, real* cost, *ThreadLocalRand::getSeed() = FLAGS_seed; vecW.randnorm(0, 0.1); + vecMomentum.randnorm(0, 0.1); trainer.startTrain(); for (int i = 0; i < num_passes; ++i) { @@ -140,6 +139,14 @@ TEST(RecurrentGradientMachine, rnn_multi_input) { } } +TEST(RecurrentGradientMachine, rnn_multi_unequalength_input) { + for (bool useGpu : {false, true}) { + test("gserver/tests/sequence_rnn_multi_unequalength_inputs.conf", + "gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf", + 1e-6, useGpu); + } +} + int main(int argc, char** argv) { if (paddle::version::isWithPyDataProvider()) { if (!paddle::version::isWithGpu()) { diff --git a/paddle/gserver/tests/test_RecurrentLayer.cpp b/paddle/gserver/tests/test_RecurrentLayer.cpp index 9b933b153d158..1c8497e8c526f 100644 --- a/paddle/gserver/tests/test_RecurrentLayer.cpp +++ b/paddle/gserver/tests/test_RecurrentLayer.cpp @@ -369,7 +369,7 @@ TEST(Layer, LstmLayer) { LayerConfig layerConfig; layerConfig.set_type("lstmemory"); layerConfig.set_active_type("relu"); - layerConfig.set_active_state_type("sigmoid"); + layerConfig.set_active_state_type("tanh"); layerConfig.set_active_gate_type("sigmoid"); layerConfig.add_inputs(); diff --git a/paddle/math/BaseMatrix.cu b/paddle/math/BaseMatrix.cu index 8b888b1ee5e46..54448bdb5a9bb 100644 --- a/paddle/math/BaseMatrix.cu +++ b/paddle/math/BaseMatrix.cu @@ -625,7 +625,10 @@ void BaseMatrixT::squareDerivative(BaseMatrixT& b) { applyBinary(binary::SquareDerivative(), b); } -DEFINE_MATRIX_BINARY_OP(Tanh, b = 2.0 / (1.0 + exp(-2 * a)) - 1.0); +DEFINE_MATRIX_BINARY_OP(Tanh, + T tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + b = 2.0 / (1.0 + std::exp(tmp)) - 1.0); template<> void BaseMatrixT::tanh(BaseMatrixT& b) { applyBinary(binary::Tanh(), b); @@ -1448,6 +1451,8 @@ int BaseMatrixT::applyRow(Agg agg, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(height_, numRows); + CHECK_EQ(width_, 1UL); aggregate(agg, base::unary::identity(), base::binary::second(), b, numRows, numCols, offset, false_type(), true_type() /*aAsColVector*/); @@ -1460,18 +1465,69 @@ int BaseMatrixT::applyRow(Agg agg, Saver sv, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(height_, numRows); + CHECK_EQ(width_, 1UL); aggregate(agg, base::unary::identity(), sv, b, numRows, numCols, offset, false_type(), true_type() /*aAsColVector*/); return 0; } +template<> +template +int BaseMatrixT::applyRow( + Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b) { + if (scaleDest != 0) { + applyRow(agg, base::binary::add2(scaleDest, scaleAgg), b); + } else { + applyRow(agg, base::binary::second(), b); + if (scaleAgg != 1) { + mulScalar(scaleAgg); + } + } + return 0; +} + +template<> +template +int BaseMatrixT::applyRow(Agg agg, Op op, Saver sv, + BaseMatrixT& b, BaseMatrixT& c) { + MatrixOffset offset(0, 0, 0, 0, 0, 0); + int numRows = b.height_; + int numCols = b.width_; + CHECK_EQ(height_, numRows); + CHECK_EQ(width_, 1UL); + CHECK_EQ(c.height_, numRows); + CHECK_EQ(c.width_, numCols); + aggregate(agg, op, sv, + b, c, numRows, numCols, offset, + false_type(), true_type() /*aAsColVector*/); + return 0; +} + +template<> +template +int BaseMatrixT::applyRow(Agg agg, Op op, real scaleDest, real scaleAgg, + BaseMatrixT& b, BaseMatrixT& c) { + if (scaleDest != 0) { + applyRow(agg, op, base::binary::add2(scaleDest, scaleAgg), b, c); + } else { + applyRow(agg, op, base::binary::second(), b, c); + if (scaleAgg != 1) { + mulScalar(scaleAgg); + } + } + return 0; +} + template<> template int BaseMatrixT::applyCol(Agg agg, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(width_, numCols); + CHECK_EQ(height_, 1UL); aggregate(agg, base::unary::identity(), base::binary::second(), b, numRows, numCols, offset, true_type() /*aAsRowVector*/, false_type()); @@ -1484,6 +1540,8 @@ int BaseMatrixT::applyCol(Agg agg, Saver sv, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(width_, numCols); + CHECK_EQ(height_, 1UL); aggregate(agg, base::unary::identity(), sv, b, numRows, numCols, offset, true_type() /*aAsRowVector*/, false_type()); @@ -1491,8 +1549,23 @@ int BaseMatrixT::applyCol(Agg agg, Saver sv, BaseMatrixT& b) { } template<> -void BaseMatrixT::sumRows(BaseMatrixT& b) { - applyRow(aggregate::sum(), b); +template +int BaseMatrixT::applyCol( + Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b) { + if (scaleDest != 0) { + applyCol(agg, base::binary::add2(scaleDest, scaleAgg), b); + } else { + applyCol(agg, base::binary::second(), b); + if (scaleAgg != 1) { + mulScalar(scaleAgg); + } + } + return 0; +} + +template<> +void BaseMatrixT::sumRows(BaseMatrixT& b, real scaleSum, real scaleDest) { + applyRow(aggregate::sum(), scaleDest, scaleSum, b); } template<> @@ -1521,18 +1594,22 @@ void BaseMatrixT::minCols(BaseMatrixT& b) { } template<> -void BaseMatrixT::sumCols(BaseMatrixT& b, real scale) { - applyCol(aggregate::sum(), base::binary::add2(1.0, scale), b); +void BaseMatrixT::sumCols(BaseMatrixT& b, real scaleSum, real scaleDest) { + applyCol(aggregate::sum(), scaleDest, scaleSum, b); } template<> -void BaseMatrixT::sumOfSquares(BaseMatrixT& b, BaseMatrixT& c) { - int numRows = b.height_; - int numCols = b.width_; - MatrixOffset offset(0, 0, 0, 0, 0, 0); - aggregate(aggregate::sum(), base::binary::squaredDiff(), base::binary::add(), - b, c, numRows, numCols, offset, false_type(), - true_type() /*aAsColVector*/); +void BaseMatrixT::sumOfSquaredDiffs( + BaseMatrixT& b, BaseMatrixT& c, real scaleSum, real scaleDest) { + applyRow(aggregate::sum(), base::binary::squaredDiff(), + scaleDest, scaleSum, b, c); +} + +template<> +void BaseMatrixT::sumOfProducts( + BaseMatrixT& b, BaseMatrixT& c, real scaleSum, real scaleDest) { + applyRow(aggregate::sum(), base::binary::mul(), + scaleDest, scaleSum, b, c); } template class BaseMatrixT; diff --git a/paddle/math/BaseMatrix.h b/paddle/math/BaseMatrix.h index 2dd2c2c7a9b98..3a91fdc3c30c5 100644 --- a/paddle/math/BaseMatrix.h +++ b/paddle/math/BaseMatrix.h @@ -305,6 +305,23 @@ class BaseMatrixT { template int applyRow(Agg agg, BaseMatrixT& b); + /** + * a aggregate expression that apply each row of matrix b. + * + * @code + * for each row i & 0 <= j < b.width_, do: + * dst = agg(op(b[i*ldb + j], c[i*ldc + j]) + * this[i] = sv(this[i], dst) + * @endcode + */ + template + int applyRow(Agg agg, Op op, Saver sv, BaseMatrixT& b, BaseMatrixT& c); + + // Same as the above with the special handing of sv=add2(scaleDest, scaleAgg) + template + int applyRow(Agg agg, Op op, real scaleDest, real scaleAgg, + BaseMatrixT& b, BaseMatrixT& c); + /** * a aggregate expression that apply each row of matrix b. * @@ -317,6 +334,10 @@ class BaseMatrixT { template int applyRow(Agg agg, Saver sv, BaseMatrixT& b); + // Same as the above with the special handing of sv=add2(scaleDest, scaleAgg) + template + int applyRow(Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b); + /** * a aggregate expression that apply each column of matrix b. * @@ -340,6 +361,10 @@ class BaseMatrixT { template int applyCol(Agg agg, Saver sv, BaseMatrixT& b); + // Same as the above with the special handing of sv=add2(scaleDest, scaleAgg) + template + int applyCol(Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b); + bool useGpu() const { return useGpu_; } const T* rowBuf(size_t row) const { return data_ + width_ * row; } @@ -920,7 +945,9 @@ class BaseMatrixT { void addRowScale(size_t cCol, BaseMatrixT& b, BaseMatrixT& c); /// calculate the sum of each row of the matrix b. - void sumRows(BaseMatrixT& b); + /// this_i = scaleDest * this_i + scaleSum * \sum_j b_{ij} + void sumRows(BaseMatrixT& b, T scaleSum, T scaleDest); + /// calculate the maximum value of each row of the matrix b. void maxRows(BaseMatrixT& b); /// calculate the minimum value of each row of the matrix b. @@ -932,10 +959,18 @@ class BaseMatrixT { void maxCols(BaseMatrixT& b); /// calculate the minimum value of each column of the matrix b. void minCols(BaseMatrixT& b); - void sumCols(BaseMatrixT& b, T scale); - /// calculate the sum of each row of (b - c)^2. - void sumOfSquares(BaseMatrixT& b, BaseMatrixT& c); + /// calculate the sum of each column of the matrix b. + /// this_i = scaleDest * this_i + scaleSum * \sum_j b_{ji} + void sumCols(BaseMatrixT& b, T scaleSum, T scaleDest); + + /// this_i = scaleDest * this_i + scaleSum * \sum_j (b_{ij} - c_{ij})^2 + void sumOfSquaredDiffs(BaseMatrixT& b, BaseMatrixT& c, + T scaleSum, T scaleDest); + + /// this_i = scaleDest * this_i + scaleSum * \sum_j b_{ij} * c_{ij} + void sumOfProducts(BaseMatrixT& b, BaseMatrixT& c, + T scaleSum, T scaleDest); /** * @code diff --git a/paddle/math/CMakeLists.txt b/paddle/math/CMakeLists.txt index db305812a7c03..93b1bf46a1007 100644 --- a/paddle/math/CMakeLists.txt +++ b/paddle/math/CMakeLists.txt @@ -23,7 +23,7 @@ if(NOT WITH_GPU) add_library(paddle_math STATIC ${MATH_SOURCES}) else() - add_paddle_culib(paddle_math ${MATH_SOURCES}) + cuda_add_library(paddle_math ${MATH_SOURCES}) endif() diff --git a/paddle/math/CpuSparseMatrix.cpp b/paddle/math/CpuSparseMatrix.cpp index 842efdbe3d77e..64ee124a5613a 100644 --- a/paddle/math/CpuSparseMatrix.cpp +++ b/paddle/math/CpuSparseMatrix.cpp @@ -409,9 +409,6 @@ void CpuSparseMatrix::setRow(size_t row, size_t colNum, if (format_ == SPARSE_CSR) { CHECK_LT(row, height_); CHECK(NULL != cols); - for (size_t i = row; i < height_; i++) { - CHECK_EQ(rows_[i + 1], rows_[i]); - } if (0 == row) { rows_[row] = 0; } diff --git a/paddle/math/MathFunctions.cpp b/paddle/math/MathFunctions.cpp index da493379e3a37..e0b2a2bb5b2cd 100644 --- a/paddle/math/MathFunctions.cpp +++ b/paddle/math/MathFunctions.cpp @@ -39,6 +39,46 @@ void gemm(const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, beta, C, ldc); } +template<> +int getrf(const CBLAS_ORDER order, const int M, const int N, + float *A, const int lda, int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_sgetrf(order, M, N, A, lda, ipiv); +#else + return LAPACKE_sgetrf(order, M, N, A, lda, ipiv); +#endif +} + +template<> +int getrf(const CBLAS_ORDER order, const int M, const int N, + double *A, const int lda, int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_dgetrf(order, M, N, A, lda, ipiv); +#else + return LAPACKE_dgetrf(order, M, N, A, lda, ipiv); +#endif +} + +template<> +int getri(const CBLAS_ORDER order, const int N, float *A, + const int lda, const int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_sgetri(order, N, A, lda, ipiv); +#else + return LAPACKE_sgetri(order, N, A, lda, ipiv); +#endif +} + +template<> +int getri(const CBLAS_ORDER order, const int N, double *A, + const int lda, const int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_dgetri(order, N, A, lda, ipiv); +#else + return LAPACKE_dgetri(order, N, A, lda, ipiv); +#endif +} + template<> void axpy(const int n, const float alpha, const float* x, float* y) { cblas_saxpy(n, alpha, x, 1, y, 1); @@ -160,7 +200,10 @@ void vLog1p(const int n, const T* a, T* r) { binary::vLog1p(), const_cast(a), r, 1, n, n, n); } -DEFINE_MATRIX_BINARY_OP(vTanh, b = 2.0 / (1.0 + std::exp(-2 * a)) - 1.0); +DEFINE_MATRIX_BINARY_OP(vTanh, + T tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + b = 2.0 / (1.0 + std::exp(tmp)) - 1.0); template void vTanh(const int n, const T* a, T* r) { hl_cpu_apply_binary_op, 0, 0>( diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index 43075977dc9ce..29c07467c7bac 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -17,10 +17,18 @@ limitations under the License. */ #ifdef PADDLE_USE_MKL #include +#include #else extern "C" { #include } +#ifdef PADDLE_USE_ATLAS +extern "C" { +#include +} +#else +#include +#endif #endif #include @@ -34,6 +42,14 @@ void gemm(const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const T* B, const int ldb, const T beta, T* C, const int ldc); +template +int getrf(const CBLAS_ORDER Order, const int M, const int N, + T *A, const int lda, int *ipiv); + +template +int getri(const CBLAS_ORDER Order, const int N, T *A, + const int lda, const int *ipiv); + template void axpy(const int n, const T alpha, const T* x, T* y); @@ -64,4 +80,3 @@ void vTanh(const int n, const T* a, T* r); } // namespace paddle #endif // MATHFUNCTIONS_H_ - diff --git a/paddle/math/MathUtils.cpp b/paddle/math/MathUtils.cpp index 5b78ab1b07bda..548f17936381c 100644 --- a/paddle/math/MathUtils.cpp +++ b/paddle/math/MathUtils.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "MathUtils.h" #include #include "paddle/utils/Logging.h" @@ -24,11 +23,7 @@ namespace paddle { * major is rows and minor is cols, according to * major value to initialize minor value" */ -void sparseRand(int* major, - int* minor, - int nnz, - int majorLen, - int minorMax, +void sparseRand(int* major, int* minor, int nnz, int majorLen, int minorMax, bool useGpu) { CHECK(size_t(nnz) > size_t(1)); int* cpuMajor; @@ -72,5 +67,30 @@ void sparseRand(int* major, } } +int outputSize(int imageSize, int filterSize, int padding, int stride, + bool caffeMode) { + int outputSize; + if (!caffeMode) { + outputSize = + (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; + } else { + outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; + } + CHECK_GE(outputSize, 1); + return outputSize; +} + +int imageSize(int outputSize, int filterSize, int padding, int stride, + bool caffeMode) { + int imageSize; + if (!caffeMode) { + imageSize = + (outputSize - 1) * stride + filterSize - 2 * padding - stride + 1; + } else { + imageSize = (outputSize - 1) * stride + filterSize - 2 * padding; + } + CHECK_GE(imageSize, 1); + return imageSize; +} } // namespace paddle diff --git a/paddle/math/MathUtils.h b/paddle/math/MathUtils.h index 83375022abbe2..91683dc3e9144 100644 --- a/paddle/math/MathUtils.h +++ b/paddle/math/MathUtils.h @@ -44,4 +44,27 @@ namespace paddle { void sparseRand(int* major, int* minor, int nnz, int majorLen, int minorMax, bool useGpu); +/** + * Calculate output size based on caffeMode_. + * - input(+padding): 0123456789 + * - imageSize(+padding) = 10; + * - filterSize = 3; + * - stride = 2; + * - caffeMode is true: + - output: (012), (234), (456), (678) + - outputSize = 4; + * - caffeMode is false: + * - output: (012), (234), (456), (678), (9) + * - outputSize = 5; + */ +int outputSize(int imageSize, int filterSize, int padding, int stride, + bool caffeMode); + +/** + * Calculate image size based on output size and caffeMode_. + * It is the reverse function of outputSize() + */ +int imageSize(int outputSize, int filterSize, int padding, int stride, + bool caffeMode); + } // namespace paddle diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index e351bede724ac..706a598d0c337 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -13,19 +13,20 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "Matrix.h" +#include "MathFunctions.h" #include "SparseMatrix.h" #include "SparseRowMatrix.h" -#include "MathFunctions.h" -#include #include #include +#include -#include "paddle/utils/Logging.h" #include +#include "hl_cnn.h" #include "hl_gpu.h" #include "hl_table_apply.h" #include "hl_top_k.h" +#include "paddle/utils/Logging.h" #include "paddle/utils/ThreadLocal.h" @@ -42,9 +43,9 @@ inline real _safelog(real a) { return a > 0.0f ? std::log(a) : -40.0f; } Matrix::Matrix(MemoryHandlePtr memHandle, size_t height, size_t width, bool trans, bool use_gpu) : BaseMatrix( - height, width, - memHandle ? (reinterpret_cast(memHandle->getBuf())) : nullptr, - trans, use_gpu) { + height, width, + memHandle ? (reinterpret_cast(memHandle->getBuf())) : nullptr, + trans, use_gpu) { elementCnt_ = width * height; memoryHandle_ = memHandle; } @@ -95,7 +96,7 @@ MatrixPtr Matrix::create(MemoryHandlePtr memHandle, size_t height, size_t width, if (auto gpuHandle = std::dynamic_pointer_cast(memHandle)) { return std::make_shared(gpuHandle, height, width, trans); } else if (auto cpuHandle = - std::dynamic_pointer_cast(memHandle)) { + std::dynamic_pointer_cast(memHandle)) { return std::make_shared(cpuHandle, height, width, trans); } else { LOG(FATAL) << "Wrong"; @@ -187,6 +188,15 @@ MatrixPtr Matrix::subMatrix(size_t startRow, size_t endRow, size_t startCol, trans_, useGpu_); } +void Matrix::setDiag(real value) { + CHECK(data_ != NULL); + CHECK_EQ(height_, width_); + + zeroMem(); + BaseMatrix diag(height_, 1, stride_ + 1, data_, false, useGpu_); + diag.assign(value); +} + GpuMatrix::GpuMatrix(size_t height, size_t width, bool trans) : Matrix(std::make_shared(height * width * sizeof(real)), height, width, trans, true) {} @@ -202,6 +212,7 @@ void GpuMatrix::resetOne() { CHECK(data_ != NULL); one(); } + void GpuMatrix::resize(size_t newHeight, size_t newWidth) { size_t newSize = newHeight * newWidth; if (NULL == memoryHandle_.get() || @@ -231,7 +242,7 @@ real GpuMatrix::getSum() { void GpuMatrix::accumulateColSum(Matrix& src) { CHECK_EQ(getWidth(), src.getWidth()); CHECK_EQ(getHeight(), (size_t)1); - sumCols(src, 1.0); + sumCols(src, 1.0, 1.0); } real GpuMatrix::getAbsSum() { @@ -282,13 +293,13 @@ void GpuMatrix::copyFrom(const IVector& src) { copyFrom(matrix); } -void GpuMatrix::copyByRowIndex(Matrix& b, IVector& rowIndex) { +void GpuMatrix::copyByRowIndex(Matrix& b, const IVector& rowIndex) { size_t height = getHeight(); size_t width = getWidth(); CHECK_EQ(b.getWidth(), width); real* dst = getData(); real* src = b.getData(); - int* index = rowIndex.getData(); + const int* index = rowIndex.getData(); hl_sequence2batch_copy(dst, src, index, width, height, true); } @@ -335,25 +346,66 @@ void GpuMatrix::transpose(MatrixPtr matTrans, bool memAlloc) { hl_matrix_transpose(data, dataTrans, height_, width_, lda, ldc); } + +MatrixPtr GpuMatrix::getInverse() { + MatrixPtr matInv; + inverse(matInv, true); + return matInv; +} + +void GpuMatrix::inverse(MatrixPtr matInv, bool memAlloc) { + CHECK_EQ(height_, width_); + + if (memAlloc) { + matInv = std::make_shared(height_, width_); + } else { + CHECK(matInv != NULL); + } + + real* data = getData(); + real* dataInv = matInv->getData(); + int lda = getStride(); + int ldc = matInv->getStride(); + + hl_matrix_inverse(data, dataInv, height_, lda, ldc); +} + void GpuMatrix::addBias(Matrix& b, real scale) { CHECK(b.getHeight() == 1) << "the Bias should be a vector"; BaseMatrix::addBias(b, scale); } +void GpuMatrix::addSharedBias(Matrix& b, real scale) { + CHECK(b.getHeight() == 1) << "the Bias should be a vector"; + CHECK_LE(b.getWidth(), getWidth()); + CHECK_EQ(getWidth() % b.getWidth(), 0UL); + hl_matrix_add_shared_bias(getData(), b.getData(), b.getWidth(), + getHeight(), getWidth(), scale); +} + + void GpuMatrix::collectBias(Matrix& a, real scale) { CHECK_EQ(getHeight(), (size_t)1); CHECK_EQ(width_, a.getWidth()); - GpuSparseMatrix* sMatPtr = dynamic_cast(&a); + GpuSparseMatrix* sMatPtr = dynamic_cast(&a); if (!sMatPtr) { - sumCols(a, scale); + sumCols(a, /* scaleSum= */scale, /* scaleDest= */1); } else { real* data = getData(); hl_sparse_matrix_s A_d = sMatPtr->sMatrix_.get(); - hl_sparse_matrix_column_sum(data, A_d, sMatPtr->getHeight(), - width_, scale); + hl_sparse_matrix_column_sum(data, A_d, sMatPtr->getHeight(), width_, scale); } } + +void GpuMatrix::collectSharedBias(Matrix& a, real scale) { + CHECK_EQ(getHeight(), (size_t)1); + CHECK_EQ(a.getWidth() % getWidth(), 0UL); + hl_matrix_collect_shared_bias(getData(), a.getData(), getWidth(), + a.getHeight(), a.getWidth(), scale); +} + + void GpuMatrix::sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { @@ -401,8 +453,8 @@ void GpuMatrix::mul(const GpuMatrix& a, const GpuMatrix& b, real scaleAB, hl_trans_op_t transa = !a.isTransposed() ? HPPL_OP_N : HPPL_OP_T; hl_trans_op_t transb = !b.isTransposed() ? HPPL_OP_N : HPPL_OP_T; - hl_matrix_mul(A_d, transa, B_d, transb, C_d, dimM, dimN, dimK, - scaleAB, scaleT, lda, ldb, ldc); + hl_matrix_mul(A_d, transa, B_d, transb, C_d, dimM, dimN, dimK, scaleAB, + scaleT, lda, ldb, ldc); } void GpuMatrix::mul(const GpuSparseMatrix& a, const GpuMatrix& b, real scaleAB, @@ -423,8 +475,8 @@ void GpuMatrix::mul(const GpuSparseMatrix& a, const GpuMatrix& b, real scaleAB, hl_sparse_matrix_s A_d = a.sMatrix_.get(); real* B_d = b.data_; real* C_d = data_; - hl_matrix_csr_mul_dense(A_d, transA, B_d, HPPL_OP_N, C_d, height_, - width_, b.height_, scaleAB, scaleT); + hl_matrix_csr_mul_dense(A_d, transA, B_d, HPPL_OP_N, C_d, height_, width_, + b.height_, scaleAB, scaleT); } void GpuMatrix::mul(const GpuMatrix& a, const GpuSparseMatrix& b, real scaleAB, @@ -445,11 +497,11 @@ void GpuMatrix::mul(const GpuMatrix& a, const GpuSparseMatrix& b, real scaleAB, << "Matrix dimensions are not equal"; } if (b.format_ == SPARSE_CSC) { - hl_matrix_dense_mul_csc(A_d, HPPL_OP_N, B_d, transB, C_d, height_, - width_, a.width_, scaleAB, scaleT); + hl_matrix_dense_mul_csc(A_d, HPPL_OP_N, B_d, transB, C_d, height_, width_, + a.width_, scaleAB, scaleT); } else { - hl_matrix_dense_mul_csr(A_d, HPPL_OP_N, B_d, transB, C_d, height_, - width_, a.width_, scaleAB, scaleT); + hl_matrix_dense_mul_csr(A_d, HPPL_OP_N, B_d, transB, C_d, height_, width_, + a.width_, scaleAB, scaleT); } } @@ -511,8 +563,8 @@ void GpuMatrix::selectRows(Matrix& table, IVector& ids) { size_t tableSize = table.getHeight(); int* index = ids.getData(); - hl_matrix_select_rows(a, stride_, table.getData(), table.stride_, - index, numSamples, tableSize, dim); + hl_matrix_select_rows(a, stride_, table.getData(), table.stride_, index, + numSamples, tableSize, dim); #endif } @@ -529,15 +581,15 @@ void GpuMatrix::addToRows(Matrix& table, IVector& ids) { size_t tableSize = table.getHeight(); int* index = ids.getData(); - hl_matrix_add_to_rows(table.getData(), table.stride_, a, stride_, - index, numSamples, tableSize, dim); + hl_matrix_add_to_rows(table.getData(), table.stride_, a, stride_, index, + numSamples, tableSize, dim); #endif } void GpuMatrix::colMerge(Matrix& src) { CHECK(src.height_ == height_); if (!trans_ && !src.trans_) { - sumRows(src); + sumRows(src, /* scaleSum= */1, /* scaleDest= */0); } else { LOG(FATAL) << "Is not supported"; } @@ -547,7 +599,7 @@ void GpuMatrix::rowSum(Matrix& sum) { CHECK_EQ(sum.getHeight(), getHeight()); CHECK_EQ(sum.getWidth(), (size_t)1); - sum.sumRows(*this); + sum.sumRows(*this, /* scaleSum= */1, /* scaleDest= */0); } void GpuMatrix::rowMax(Matrix& max) { @@ -565,13 +617,8 @@ void GpuMatrix::rowMax(IVector& maxIds, Matrix& maxVal) { CHECK_EQ(maxIds.getSize(), numSamples * beam); CHECK_EQ(maxVal.getHeight(), numSamples); - hl_matrix_top_k(maxVal.getData(), - maxVal.getStride(), - maxIds.getData(), - this->getData(), - this->getStride(), - this->getWidth(), - beam, + hl_matrix_top_k(maxVal.getData(), maxVal.getStride(), maxIds.getData(), + this->getData(), this->getStride(), this->getWidth(), beam, numSamples); #endif } @@ -583,6 +630,42 @@ void GpuMatrix::colMax(Matrix& max) { max.maxCols(*this); } +void GpuMatrix::colMax(IVector& maxIds, Matrix& maxVal) { + LOG(FATAL) << "Is not supported"; +} + +void GpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = getWidth(); + size_t batchSize = getHeight(); + const real* input = a.getData(); + real* output = getData(); + int* idForGpu = id.getData(); + + hl_maxout_forward(input, output, idForGpu, batchSize, size, size / channels, + groups); +} + +void GpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = a.getWidth(); + size_t batchSize = getHeight(); + real* input = getData(); + const real* output = a.getData(); + const int* idForGpu = id.getData(); + + hl_maxout_backward(input, output, idForGpu, batchSize, size, size / channels, + groups); +} + /*calulate the error of classification */ void GpuMatrix::classificationError(MatrixPtr output, IVectorPtr label) { GpuMatrixPtr output_ptr = std::dynamic_pointer_cast(output); @@ -596,8 +679,8 @@ void GpuMatrix::classificationError(MatrixPtr output, IVectorPtr label) { real* recResult_d = data_; int* label_d = label_ptr->getData(); - hl_matrix_classification_error(output_d, label_d, recResult_d, - height_, output_ptr->width_); + hl_matrix_classification_error(output_d, label_d, recResult_d, height_, + output_ptr->width_); } /* copy -log(output[i * width + label]) to this->data[i] */ @@ -666,8 +749,7 @@ void GpuMatrix::sequenceSoftmax(Matrix& output, const IVector& index) { real* outputData = output.getData(); auto starts = index.getData(); int numSequences = index.getSize() - 1; - hl_sequence_softmax_forward(inputData, outputData, - starts, numSequences); + hl_sequence_softmax_forward(inputData, outputData, starts, numSequences); } void GpuMatrix::softmaxDerivative(Matrix& output, Matrix& sftmaxSum) { @@ -681,8 +763,7 @@ void GpuMatrix::softmaxDerivative(Matrix& output, Matrix& sftmaxSum) { real* output_d = output.data_; real* sftmaxSum_d = sftmaxSum.data_; real* grad_d = data_; - hl_matrix_softmax_derivative(grad_d, output_d, sftmaxSum_d, height_, - width_); + hl_matrix_softmax_derivative(grad_d, output_d, sftmaxSum_d, height_, width_); } void GpuMatrix::softmaxBackward(Matrix& outputV) { @@ -709,7 +790,8 @@ void GpuMatrix::sumOfSquares(Matrix& output, Matrix& label) { LOG(FATAL) << "not supported: GpuSparseMatrix as label"; } - BaseMatrix::sumOfSquares(output, label); + BaseMatrix::sumOfSquaredDiffs(output, label, + /* scaleSum= */1, /* scaleDest= */1); } void GpuMatrix::sumOfSquaresBp(Matrix& outputV, Matrix& label) { @@ -733,7 +815,7 @@ void GpuMatrix::scaledTanh(Matrix& output, real p1, real p2) { } void GpuMatrix::cosSim(Matrix& output1, Matrix& output2, real scale) { CHECK(output1.useGpu_ == true && output2.useGpu_ == true) - << "Matrix type are not equal"; + << "Matrix type are not equal"; size_t numSamples = getHeight(); size_t dim = output1.getWidth(); CHECK_EQ(getWidth(), 1UL); @@ -742,15 +824,15 @@ void GpuMatrix::cosSim(Matrix& output1, Matrix& output2, real scale) { real* out = getData(); real* x = output1.getData(); real* y = output2.getData(); - hl_cossim(out, x, y, - dim, output1.getHeight(), output2.getHeight(), scale); + hl_cossim(out, x, y, dim, output1.getHeight(), output2.getHeight(), scale); } void GpuMatrix::cosSimDerivative(Matrix& output, Matrix& prevOut1, Matrix& prevOut2, Matrix& prevGrad1, Matrix& prevGrad2, real scale) { CHECK(output.useGpu_ == true && prevOut1.useGpu_ == true && prevOut2.useGpu_ == true && prevGrad1.useGpu_ == true && - prevGrad2.useGpu_ == true) << "Matrix type are not equal"; + prevGrad2.useGpu_ == true) + << "Matrix type are not equal"; CHECK_EQ(getWidth(), 1UL); CHECK_EQ(output.getWidth(), 1UL); @@ -770,9 +852,8 @@ void GpuMatrix::cosSimDerivative(Matrix& output, Matrix& prevOut1, real* prevOutY = prevOut2.getData(); real* prevGradX = prevGrad1.getData(); real* prevGradY = prevGrad2.getData(); - hl_cossim_derivative(grad, out, prevOutX, prevOutY, - prevGradX, prevGradY, dim, - prevOut1.getHeight(), prevOut2.getHeight(), scale); + hl_cossim_derivative(grad, out, prevOutX, prevOutY, prevGradX, prevGradY, dim, + prevOut1.getHeight(), prevOut2.getHeight(), scale); } void GpuMatrix::randomizeUniform() { @@ -823,8 +904,8 @@ void GpuMatrix::check(std::ostream& os, Matrix& refMat, bool printDiff) { void GpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, int channels, int blockH, int blockW, int strideH, - int strideW, int paddingH, int paddingW, - int outputH, int outputW) { + int strideW, int paddingH, int paddingW, int outputH, + int outputW) { CHECK(feature.useGpu_ == true) << "Matrix type are not equal"; CHECK_EQ(size_t(feaImgHeight * feaImgWidth * channels), @@ -834,17 +915,16 @@ void GpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, size_t elemCnt = outputH * outputW * blockH * blockW * channels; CHECK_EQ(elemCnt, height_ * width_) << "Matrix dimensions are not equal"; - hl_expand_feature2col(feature.getData(), channels, feaImgHeight, - feaImgWidth, blockH, blockW, strideH, strideW, - paddingH, paddingW, outputH, outputW, - getData()); + hl_expand_feature2col(feature.getData(), channels, feaImgHeight, feaImgWidth, + blockH, blockW, strideH, strideW, paddingH, paddingW, + outputH, outputW, getData()); } void GpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, int thisImgWidth, int channels, int blockH, int blockW, int strideH, int strideW, int paddingH, - int paddingW, int outputH, int outputW, - real alpha, real beta) { + int paddingW, int outputH, int outputW, real alpha, + real beta) { CHECK(expandFeat.useGpu_ == true) << "Matrix type are not equal"; CHECK_EQ(size_t(thisImgHeight * thisImgWidth * channels), getHeight() * getWidth()) @@ -853,16 +933,17 @@ void GpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, size_t elemCnt = outputH * outputW * blockW * blockH * channels; CHECK(elemCnt == expandFeat.getHeight() * expandFeat.getWidth()) << "Matrix dimensions are not equal"; - hl_shrink_col2feature( - expandFeat.getData(), channels, thisImgHeight, thisImgWidth, blockH, - blockW, strideH, strideW, paddingH, paddingW, outputH, outputW, - getData(), alpha, beta); + hl_shrink_col2feature(expandFeat.getData(), channels, thisImgHeight, + thisImgWidth, blockH, blockW, strideH, strideW, + paddingH, paddingW, outputH, outputW, getData(), alpha, + beta); } void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, size_t channels, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -873,15 +954,17 @@ void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(height_ == inputMat.getHeight()); CHECK(width_ == outputH * outputW * channels); - hl_maxpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, stride, start, data_); + hl_maxpool_forward(frameNum, inputData, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, data_, getStride()); } void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, - size_t sizeX, int start, size_t stride, - size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + size_t sizeX, size_t sizeY, size_t strideH, + size_t strideW, size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { CHECK(inputMat.useGpu_ == true && outGrad.useGpu_ == true && outV.useGpu_ == true) << "Matrix type are not equal"; @@ -899,15 +982,17 @@ void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, CHECK(outGrad.getHeight() == outV.getHeight() && outGrad.getWidth() == outV.getWidth()); - hl_maxpool_backward(frameNum, inputData, outData, outDiff, channels, - height, width, outputH, outputW, sizeX, stride, - start, data_, scaleTargets, scaleOutput); + hl_maxpool_backward(frameNum, inputData, outData, outDiff, channels, height, + width, outputH, outputW, sizeX, sizeY, strideH, strideW, + paddingH, paddingW, scaleTargets, scaleOutput, data_, + outGrad.getStride()); } void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, size_t channels, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -918,14 +1003,17 @@ void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(height_ == inputMat.getHeight()); CHECK(width_ == outputH * outputW * channels); - hl_avgpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, stride, start, data_); + hl_avgpool_forward(frameNum, inputData, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, data_, getStride()); } void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, - size_t imgSizeW, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + size_t imgSizeW, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, size_t outputH, + size_t outputW, real scaleTargets, + real scaleOutput, size_t paddingH, + size_t paddingW) { CHECK(outGrad.useGpu_ == true) << "Matrix type are not equal"; real* outDiff = outGrad.getData(); @@ -937,9 +1025,10 @@ void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, CHECK(height_ == outGrad.getHeight()); CHECK(outGrad.getWidth() == outputH * outputW * channels); - hl_avgpool_backward(frameNum, outDiff, channels, height, width, - outputH, outputW, sizeX, stride, start, data_, - scaleTargets, scaleOutput); + hl_avgpool_backward(frameNum, outDiff, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, scaleTargets, scaleOutput, data_, + outGrad.getStride()); } void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, @@ -954,8 +1043,8 @@ void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, CHECK(denoms.getHeight() == input.getHeight() && denoms.getWidth() == input.getWidth() && input.getHeight() == height_ && input.getWidth() == width_); - hl_CMRNorm_forward(num, input.getData(), denoms.getData(), data_, - channels, height, width, sizeX, scale, -pow); + hl_CMRNorm_forward(num, input.getData(), denoms.getData(), data_, channels, + height, width, sizeX, scale, -pow); } void GpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, @@ -975,13 +1064,11 @@ void GpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, denoms.getWidth() == localGrad.getWidth()); hl_CMRNorm_backward(num, preOutV.getData(), denoms.getData(), - localOutV.getData(), localGrad.getData(), data_, - channels, height, width, sizeX, -pow, - 2.0f * pow * scale); + localOutV.getData(), localGrad.getData(), data_, channels, + height, width, sizeX, -pow, 2.0f * pow * scale); } -void GpuMatrix::maxSequenceForward(Matrix& input, - const IVector& sequence, +void GpuMatrix::maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&input)); CHECK(dynamic_cast(&sequence)); @@ -998,12 +1085,11 @@ void GpuMatrix::maxSequenceForward(Matrix& input, CHECK_EQ(numSequences, sequence.getSize() - 1); CHECK_EQ(numSequences * dim, index.getSize()); - hl_max_sequence_forward(inputData, starts, outData, maxIndex, - numSequences, dim); + hl_max_sequence_forward(inputData, starts, outData, maxIndex, numSequences, + dim); } -void GpuMatrix::maxSequenceBackward(Matrix& outputGrad, - const IVector& sequence, +void GpuMatrix::maxSequenceBackward(Matrix& outputGrad, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&outputGrad)); CHECK(dynamic_cast(&sequence)); @@ -1060,9 +1146,8 @@ void GpuMatrix::contextProjectionBackwardData(MatrixPtr inputGrad, real* inGrad = inputGrad->getData(); const int* starts = sequence.getData(); - hl_context_projection_backward_data(outGrad, starts, inGrad, - numSequences, inputDim, - contextLength, contextStart); + hl_context_projection_backward_data(outGrad, starts, inGrad, numSequences, + inputDim, contextLength, contextStart); } void GpuMatrix::contextProjectionBackwardWeight(MatrixPtr weightGrad, @@ -1082,9 +1167,9 @@ void GpuMatrix::contextProjectionBackwardWeight(MatrixPtr weightGrad, real* wtGrad = weightGrad->getData(); const int* starts = sequence.getData(); - hl_context_projection_backward_weight( - outGrad, starts, wtGrad, numSequences, weightDim, totalPad, contextLength, - contextStart, beginPad); + hl_context_projection_backward_weight(outGrad, starts, wtGrad, numSequences, + weightDim, totalPad, contextLength, + contextStart, beginPad); } void GpuMatrix::paramReluForward(Matrix& data, Matrix& W) { @@ -1096,8 +1181,7 @@ void GpuMatrix::paramReluForward(Matrix& data, Matrix& W) { size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (W.getHeight() * W.getWidth()); real* output = getData(); - hl_param_relu_forward(output, input, w, numElements, numSamples, - partial_sum); + hl_param_relu_forward(output, input, w, numElements, numSamples, partial_sum); } void GpuMatrix::paramReluBackwardW(Matrix& oGrad, Matrix& data) { @@ -1109,8 +1193,8 @@ void GpuMatrix::paramReluBackwardW(Matrix& oGrad, Matrix& data) { size_t numElements = data.getWidth(); size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (this->getHeight() * this->getWidth()); - hl_param_relu_backward_w(wgrad, ograd, input, - numElements, numSamples, partial_sum); + hl_param_relu_backward_w(wgrad, ograd, input, numElements, numSamples, + partial_sum); } void GpuMatrix::paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { @@ -1121,14 +1205,106 @@ void GpuMatrix::paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { size_t numElements = data.getWidth(); size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (W.getHeight() * W.getWidth()); - hl_param_relu_backward_diff(ograd, input, w, diff, - numElements, numSamples, partial_sum); + hl_param_relu_backward_diff(ograd, input, w, diff, numElements, numSamples, + partial_sum); } void GpuMatrix::addColumnVector(const Matrix& b) { BaseMatrix::addColVector(const_cast(b)); } +void GpuMatrix::bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + CHECK(dynamic_cast(&in)); + + const size_t outputW = getWidth(); + const size_t outputH = getHeight(); + const size_t inputW = in.getWidth(); + const size_t inputH = in.getHeight(); + + real* outData = getData(); + const real* inData = in.getData(); + + if (inImgH == outImgW && inImgW == outImgW) { + this->copyFrom(in); + } else { + hl_bilinear_forward( + inData, inImgH, inImgW, inputH, inputW, outData, + outImgH, outImgW, outputH, outputW, numChannels, + ratioH, ratioW); + } +} + +void GpuMatrix::bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + CHECK(dynamic_cast(&out)); + + const size_t inputW = getWidth(); + const size_t inputH = getHeight(); + const size_t outputW = out.getWidth(); + const size_t outputH = out.getHeight(); + + real* inGrad = getData(); + const real* outGrad = out.getData(); + + if (outImgH == inImgH && outImgW == inImgW) { + this->add(const_cast(out)); + } else { + hl_bilinear_backward( + inGrad, inImgH, inImgW, inputH, inputW, outGrad, + outImgH, outImgW, outputH, outputW, numChannels, + ratioH, ratioW); + } +} + +void GpuMatrix::multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label) { + GpuMatrix* outputPtr = dynamic_cast(&output); + auto labelPtr = dynamic_cast(&label); + + CHECK(outputPtr && labelPtr) << "Invalid argument pointer"; + CHECK(labelPtr->format_ == SPARSE_CSR) << "Matrix format not supported"; + CHECK(height_ == outputPtr->height_ && width_ == 1 + && outputPtr->width_ == labelPtr->getWidth() + && outputPtr->height_ == labelPtr->getHeight()) + << "Matrix dimensions are not equal"; + + real* output_d = outputPtr->data_; + real* entropy_d = data_; + hl_sparse_matrix_s mat_d = labelPtr->sMatrix_.get(); + hl_matrix_multi_binary_cross_entropy( + output_d, entropy_d, mat_d, height_, outputPtr->width_); +} + +void GpuMatrix::multiBinaryLabelCrossEntropyBp(Matrix &output, Matrix &label) { + GpuMatrix* outputPtr = dynamic_cast(&output); + auto labelPtr = dynamic_cast(&label); + + CHECK(outputPtr && labelPtr) << "Invalid argument pointer"; + CHECK(labelPtr->format_ == SPARSE_CSR) << "Matrix format not supported"; + CHECK(height_ == outputPtr->height_ && width_ == outputPtr->width_ + && outputPtr->width_ == labelPtr->getWidth() + && outputPtr->height_ == labelPtr->getHeight()) + << "Matrix dimensions are not equal"; + + real* output_d = outputPtr->data_; + real* grad_d = data_; + hl_sparse_matrix_s mat_d = labelPtr->sMatrix_.get(); + hl_matrix_multi_binary_cross_entropy_bp( + output_d, grad_d, mat_d, height_, width_); +} + /** * CpuMatrix */ @@ -1263,11 +1439,11 @@ void CpuMatrix::copyFrom(const IVector& src) { } } -void CpuMatrix::copyByRowIndex(Matrix& b, IVector& rowIndex) { +void CpuMatrix::copyByRowIndex(Matrix& b, const IVector& rowIndex) { size_t height = getHeight(); size_t width = getWidth(); CHECK_EQ(b.getWidth(), width); - int* index = rowIndex.getData(); + const int* index = rowIndex.getData(); for (size_t i = 0; i < height; i++) { CHECK_LT(static_cast(index[i]), b.getHeight()); real* src = b.getData() + index[i] * width; @@ -1326,7 +1502,7 @@ void CpuMatrix::accumulateColSum(Matrix& src) { CHECK_EQ(getWidth(), src.getWidth()); CHECK_EQ(getHeight(), (size_t)1); - sumCols(src, 1.0); + sumCols(src, /* scaleSum= */1, /* scaleDest= */1); } real CpuMatrix::getAbsSum() { @@ -1369,10 +1545,51 @@ void CpuMatrix::transpose(MatrixPtr matTrans, bool memAlloc) { } } + +MatrixPtr CpuMatrix::getInverse() { + MatrixPtr matInv; + inverse(matInv, true); + return matInv; +} + +void CpuMatrix::inverse(MatrixPtr matInv, bool memAlloc) { + CHECK_EQ(height_, width_); + + if (memAlloc) { + matInv = std::make_shared(height_, width_); + } else { + CHECK(matInv != NULL); + } + + CHECK_EQ(height_, matInv->getHeight()); + CHECK_EQ(width_, matInv->getWidth()); + matInv->copyFrom(*this); + + real* data = getData(); + real* dataInv = matInv->getData(); + int ldc = matInv->getStride(); + + if (height_ == 1) { + CHECK_NE(*data, 0); + *dataInv = 1.0 / (*data); + return; + } + + /* Compute the LU decomposition of the matrix */ + std::vector ipiv(height_); + CBLAS_ORDER order = (matInv->isTransposed() ? CblasColMajor : CblasRowMajor); + int info = getrf(order, height_, height_, dataInv, ldc, ipiv.data()); + CHECK_EQ(info, 0); + + /* Compute the inverse of the matrix given its LU decompsotion */ + info = getri(order, height_, dataInv, ldc, ipiv.data()); + CHECK_EQ(info, 0); +} + void CpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, int channels, int blockH, int blockW, int strideH, - int strideW, int paddingH, int paddingW, - int outputH, int outputW) { + int strideW, int paddingH, int paddingW, int outputH, + int outputW) { CHECK(feature.useGpu_ == false) << "Matrix type are not equal"; CHECK_EQ(size_t(feaImgHeight * feaImgWidth * channels), @@ -1412,8 +1629,8 @@ void CpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, void CpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, int thisImgWidth, int channels, int blockH, int blockW, int strideH, int strideW, int paddingH, - int paddingW, int outputH, int outputW, - real alpha, real beta) { + int paddingW, int outputH, int outputW, real alpha, + real beta) { CHECK(expandFeat.useGpu_ == false) << "Matrix type are not equal"; CHECK_EQ(size_t(thisImgHeight * thisImgWidth * channels), getHeight() * getWidth()) @@ -1451,31 +1668,42 @@ void CpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, size_t channels, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { real* inputData = inputMat.getData(); real* outData = data_; size_t num = inputMat.getHeight(); size_t inWidth = imgSizeW; size_t inHeight = imgSizeH; CHECK(inHeight * inWidth == inputMat.getWidth() / channels); + CHECK_EQ(num, this->getHeight()); + CHECK_EQ(channels * outputH * outputW, this->getWidth()); + size_t outStride = getStride(); /* initialize the data_ */ - for (size_t i = 0; i < height_ * width_; i++) { - data_[i] = -FLT_MAX; + for (size_t i = 0; i < height_; i++) { + for (size_t j = 0; j < width_; j++) { + outData[i * outStride + j] = -(real)FLT_MAX; + } } /* pool max one by one */ - for (size_t n = 0; n < num; ++n) { // frame by frame + for (size_t n = 0; n < num; ++n) { // frame by frame + if (!isContiguous()) { + outData = data_ + n * outStride; + } for (size_t c = 0; c < channels; ++c) { // channel by channel for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, inHeight); - size_t wend = std::min(wstart + sizeX, inWidth); - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, inHeight); + int wend = std::min(wstart + sizeX, inWidth); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { outData[ph * outputW + pw] = std::max(outData[ph * outputW + pw], inputData[h * inWidth + w]); } @@ -1491,9 +1719,10 @@ void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, - real scaleOutput) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { size_t num = image.getHeight(); size_t channels = size_t(width_ / imgSizeH / imgSizeW); CHECK(image.getWidth() == imgSizeH * imgSizeW * channels); @@ -1505,36 +1734,49 @@ void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, real* inData = image.getData(); real* otData = outV.getData(); real* otGrad = outGrad.getData(); + + size_t outStride = outV.getStride(); + real* origOutData = otData; + real* origOutGrad = otGrad; + for (size_t n = 0; n < num; ++n) { + if (!outV.isContiguous()) { + otData = origOutData + n * outStride; + otGrad = origOutGrad + n * outStride; + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, imgSizeH); - size_t wend = std::min(wstart + sizeX, imgSizeW); - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, imgSizeH); + int wend = std::min(wstart + sizeX, imgSizeW); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { tgtGrad[h * imgSizeW + w] = scaleTargets * tgtGrad[h * imgSizeW + w] + scaleOutput * otGrad[ph * outputW + pw] * - (inData[h * imgSizeW + w] == otData[ph * outputH + pw]); + (inData[h * imgSizeW + w] == otData[ph * outputW + pw]); } } } } // offset inData += imgSizeH * imgSizeW; - otData += outputH * outputW; tgtGrad += imgSizeH * imgSizeW; + otData += outputH * outputW; otGrad += outputH * outputW; } } } void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW) { + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, size_t outputH, + size_t outputW, size_t paddingH, + size_t paddingW) { // The main loop size_t num = input.getHeight(); size_t inHeight = imgSizeH; @@ -1545,20 +1787,30 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, real* inData = input.getData(); for (size_t n = 0; n < num; ++n) { + if (!isContiguous()) { + tgtData = data_ + n * getStride(); + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, inHeight); - size_t wend = std::min(wstart + sizeX, inWidth); + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, inHeight + paddingH); + int wend = std::min(wstart + sizeX, inWidth + paddingW); + int poolSize = (hend - hstart) * (wend - wstart); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + hend = std::min(hend, static_cast(inHeight)); + wend = std::min(wend, static_cast(inWidth)); + + CHECK(poolSize); tgtData[ph * outputW + pw] = 0; // clear - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { tgtData[ph * outputW + pw] += inData[h * inWidth + w]; } } - tgtData[ph * outputW + pw] /= (hend - hstart) * (wend - wstart); + tgtData[ph * outputW + pw] /= poolSize; } } // compute offset @@ -1569,9 +1821,10 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, } void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, - size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + size_t sizeX, size_t sizeY, size_t strideH, + size_t strideW, size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { size_t num = input.getHeight(); size_t channels = input.getWidth() / outputH / outputW; CHECK(imgSizeH * imgSizeW * channels == getWidth()); @@ -1579,17 +1832,26 @@ void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, real* outData = getData(); for (size_t n = 0; n < num; ++n) { + if (!input.isContiguous()) { + inData = input.getData() + n * input.getStride(); + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, imgSizeH); - size_t wend = std::min(wstart + sizeX, imgSizeW); - size_t poolsize = (hend - hstart) * (wend - wstart); - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { - outData[h * imgSizeW + w] += inData[ph * outputW + pw] / poolsize; + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, imgSizeH + paddingH); + int wend = std::min(wstart + sizeX, imgSizeW + paddingW); + int poolSize = (hend - hstart) * (wend - wstart); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + hend = std::min(hend, static_cast(imgSizeH)); + wend = std::min(wend, static_cast(imgSizeW)); + CHECK(poolSize); + + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + outData[h * imgSizeW + w] += inData[ph * outputW + pw] / poolSize; } } } @@ -1675,8 +1937,7 @@ void CpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, * Output: output size is the number of input sequences (NOT input instances). * output[i] is set to max_{for each instance in this sequence}{input[i]} */ -void CpuMatrix::maxSequenceForward(Matrix& input, - const IVector& sequence, +void CpuMatrix::maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&input)); CHECK(dynamic_cast(&sequence)); @@ -1717,8 +1978,7 @@ void CpuMatrix::maxSequenceForward(Matrix& input, } } -void CpuMatrix::maxSequenceBackward(Matrix& outputGrad, - const IVector& sequence, +void CpuMatrix::maxSequenceBackward(Matrix& outputGrad, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&outputGrad)); CHECK(dynamic_cast(&sequence)); @@ -1906,12 +2166,30 @@ void CpuMatrix::addBias(Matrix& b, real scale) { } } +void CpuMatrix::addSharedBias(Matrix& b, real scale) { + CHECK_EQ(b.getHeight(), (size_t)1); + real* aData = getData(); + real* bData = b.getData(); + size_t numSamples = getHeight(); + size_t channel = b.getWidth(); + CHECK_EQ(getWidth() % channel, 0UL); + size_t dim = getWidth() / channel; + + for (size_t i = 0; i < numSamples; i++) { + for (size_t c = 0; c < channel; c++) { + for (size_t j = 0; j < dim; j++) { + aData[i * getStride() + c * dim + j] += scale * bData[c]; + } + } + } +} + void CpuMatrix::collectBias(Matrix& a, real scale) { CHECK_EQ(getHeight(), (size_t)1); CHECK_EQ(width_, a.getWidth()); CpuSparseMatrix* aptr = dynamic_cast(&a); if (!aptr) { - sumCols(a, scale); + sumCols(a, /* scaleSum= */scale, /* scaleDest= */1); } else { size_t nnz = aptr->getElementCnt(); int* cols = aptr->getCols(); @@ -1923,6 +2201,23 @@ void CpuMatrix::collectBias(Matrix& a, real scale) { } } +void CpuMatrix::collectSharedBias(Matrix& a, real scale) { + CHECK_EQ(getHeight(), (size_t)1); + real* B = getData(); + real* A = a.getData(); + size_t numSamples = a.getHeight(); + size_t channel = getWidth(); + CHECK_EQ(a.getWidth() % channel, 0UL); + size_t dim = a.getWidth() / channel; + for (size_t i = 0; i < numSamples; i++) { + for (size_t c = 0; c < channel; c++) { + for (size_t j = 0; j < dim; j++) { + B[c] += scale * A[i * channel * dim + c * dim + j]; + } + } + } +} + void CpuMatrix::sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { @@ -1933,7 +2228,7 @@ void CpuMatrix::sequenceAvgForward(Matrix& a, real* dst = getData(); real* src = a.getData(); const int* starts = startsPos.getData(); - MatrixPtr outMtx = Matrix::create(1, 1, false, false); + MatrixPtr outMtx = Matrix::create(nullptr, 1, width, false, false); MatrixPtr dataMtx = Matrix::create(nullptr, 1, width, false, false); for (size_t i = 0; i < height; i++) { int sequenceLength = starts[i + 1] - starts[i]; @@ -1945,13 +2240,15 @@ void CpuMatrix::sequenceAvgForward(Matrix& a, dataMtx->setData(src + starts[i] * width, sequenceLength, width); if (mode == 0) { // plain average - outMtx->sumCols(*dataMtx, (real)1 / (real)sequenceLength); + outMtx->sumCols(*dataMtx, (real)1 / (real)sequenceLength, + /* scaleDest= */1); } else if (mode == 1) { // sum instead of average - outMtx->sumCols(*dataMtx, (real)1); + outMtx->sumCols(*dataMtx, /* scaleSum= */1, /* scaleDest= */1); } else if (mode == 2) { // divide by square root of sequenceLength - outMtx->sumCols(*dataMtx, (real)1 / std::sqrt(sequenceLength)); + outMtx->sumCols(*dataMtx, (real)1 / std::sqrt(sequenceLength), + /* scaleDest= */1); } else { LOG(FATAL) << "should not reach here"; } @@ -2515,7 +2812,7 @@ void SharedCpuMatrix::mul(CpuSparseMatrix* a, CpuMatrix* b, real scaleAB, blockSeq.push_back(k); } std::shuffle(blockSeq.begin(), blockSeq.end(), - ThreadLocalRandomEngine::get()); + ThreadLocalRandomEngine::get()); } std::vector& localBufRows = *localBufRows_; int* cols = a->getCols(); @@ -2638,7 +2935,7 @@ void CpuMatrix::rowSum(Matrix& sum) { CHECK_EQ(sum.getHeight(), getHeight()); CHECK_EQ(sum.getWidth(), (size_t)1); - sum.sumRows(*this); + sum.sumRows(*this, /* scaleSum= */1, /* scaleDest= */0); } void CpuMatrix::rowMaxId(IVector& maxIds) { @@ -2707,6 +3004,95 @@ void CpuMatrix::colMax(Matrix& max) { max.maxCols(*this); } +void CpuMatrix::colMax(IVector& maxIds, Matrix& maxVal) { + CHECK(isContiguous()); + CHECK(!maxIds.useGpu() && !maxVal.useGpu()) << "Matrix type are not equal"; + size_t numSamples = getWidth(); + size_t beam = maxVal.getHeight(); + CHECK_EQ(maxIds.getSize(), numSamples * beam); + CHECK_EQ(maxVal.getWidth(), numSamples); + + real* a = getData(); + int* s = maxIds.getData(); + real* t = maxVal.getData(); + size_t dim = getHeight(); + for (size_t i = 0; i < numSamples; i++) { + std::vector> vec; + for (size_t j = 0; j < dim; j++) { + vec.push_back(std::pair(a[i + j * numSamples], j)); + } + + std::partial_sort( + vec.begin(), vec.begin() + beam, vec.end(), + [](const std::pair& l, const std::pair& r) { + return l.first > r.first; + }); + for (size_t j = 0; j < beam; j++) { + t[i + j * numSamples] = vec[j].first; + s[i + j * numSamples] = vec[j].second; + } + } +} + +void CpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = getWidth(); + size_t batchSize = getHeight(); + size_t featLen = size / channels; + const real* input = a.getData(); + int* idForCpu = id.getData(); + + MatrixPtr maxInMat, maxOutMat; + Matrix::resizeOrCreate(maxInMat, groups, size, false, false); + Matrix::resizeOrCreate(maxOutMat, 1, size, false, false); + + for (size_t batch_idx = 0; batch_idx < batchSize; ++batch_idx) { + size_t newIndex = batch_idx * size; + IVectorPtr tmpId = IVector::create(idForCpu + newIndex, size, false); + + for (size_t i = 0; i < channels; ++i) { + size_t newFeatLen = i * featLen; + for (size_t j = 0; j < groups; ++j) { + maxInMat->subMatrix(j, j + 1, newFeatLen, newFeatLen + featLen) + ->copyFrom(input + (newIndex + newFeatLen) * groups + j * featLen, + featLen); + } + } + maxInMat->colMax(*tmpId, *maxOutMat); + this->subRowMatrix(batch_idx, batch_idx + 1)->copyFrom(*maxOutMat); + } +} + +void CpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = a.getWidth(); + size_t batchSize = getHeight(); + size_t featLen = size / channels; + size_t newFeatLen = groups * featLen; + real* inputG = getData(); + const real* outG = a.getData(); + int* idForCpu = id.getData(); + + for (size_t batch_idx = 0; batch_idx < batchSize; ++batch_idx) { + size_t newIndex = batch_idx * size; + int* idData = idForCpu + newIndex; + + for (size_t i = 0; i < size; ++i) { + int gradIdx = + idData[i] * featLen + (i / featLen) * newFeatLen + i % featLen; + (inputG + newIndex * groups)[gradIdx] += (outG + newIndex)[i]; + } + } +} + void CpuMatrix::rowNormalizeL1(Matrix& out) { CHECK(!out.useGpu()); @@ -2916,9 +3302,9 @@ void CpuMatrix::sequenceSoftmax(Matrix& output, const IVector& index) { CHECK(isContiguous()); MatrixPtr inTmp = Matrix::create(nullptr, /* height= */ 1, 1, - /* trans= */ false, false); + /* trans= */ false, false); MatrixPtr outTmp = Matrix::create(nullptr, /* height= */ 1, 1, - /* trans= */ false, false); + /* trans= */ false, false); size_t numSequences = index.getSize() - 1; auto starts = index.getData(); for (size_t i = 0; i < numSequences; ++i) { @@ -3102,7 +3488,8 @@ void CpuMatrix::sumOfSquares(Matrix& output, Matrix& label) { } } - BaseMatrix::sumOfSquares(output, label); + BaseMatrix::sumOfSquaredDiffs(output, label, + /* scaleSum= */1, /* scaleDest= */1); } /* calculate the error of outputV according to label */ @@ -3188,9 +3575,7 @@ void CpuMatrix::tanh(Matrix& output) { size_t dim = getWidth(); CHECK_EQ(output.getHeight(), numSamples); CHECK_EQ(output.getWidth(), dim); - errno = 0; vTanh(numSamples * dim, getData(), output.getData()); - CHECK_EQ(errno, 0) << "vTanh error"; } void CpuMatrix::tanhDerivative(Matrix& output) { @@ -3212,10 +3597,8 @@ void CpuMatrix::softrelu(Matrix& output) { out[j] = x; } } - errno = 0; vExp(numSamples * dim, output.getData(), output.getData()); vLog1p(numSamples * dim, output.getData(), output.getData()); - CHECK_EQ(errno, 0) << "vExp+vLog1p error"; } void CpuMatrix::softreluDerivative(Matrix& output) { @@ -3230,9 +3613,7 @@ void CpuMatrix::softreluDerivative(Matrix& output) { MatrixPtr tmpMat = Matrix::create(numSamples, dim); real* tmp = tmpMat->getData(); - errno = 0; vExp(size, output.getData(), tmpMat->getData()); - CHECK_EQ(errno, 0) << "vExp error"; for (size_t i = 0; i < size; ++i) { grad[i] *= (1.0 - 1.0 / tmp[i]); @@ -3255,10 +3636,7 @@ void CpuMatrix::scaledTanh(Matrix& output, real p1, real p2) { out[i] = p2 * in[i]; } - // out = tanh(out) - errno = 0; vTanh(numSamples * dim, out, out); - CHECK_EQ(errno, 0) << "vTanh error"; // out = p1 * out for (size_t i = 0; i < numSamples * dim; ++i) { @@ -3557,6 +3935,112 @@ void CpuMatrix::classificationErrorMulti(Matrix& output, Matrix& label, } } +void CpuMatrix::bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + CHECK(dynamic_cast(&in)); + + size_t outputW = getWidth(); + size_t batchSize = getHeight(); + size_t inputW = in.getWidth(); + size_t inputH = in.getHeight(); + size_t inPosOffset = inImgH * inImgW; + size_t outPosOffset = outImgH * outImgW; + (void)(inputH); + + real* outData = getData(); + const real* inData = in.getData(); + + if (inImgH == outImgH && inImgW == outImgW) { + this->copyFrom(in); + } else { + for (size_t k = 0; k < batchSize; ++k) { // loop for batches + for (size_t i = 0; i < outImgH; ++i) { // loop for images + size_t h = ratioH * i; + size_t hid = (h < inImgH - 1) ? 1 : 0; + real h1lambda = ratioH * i - h; + real h2lambda = 1 - h1lambda; + + for (size_t j = 0; j < outImgW; ++j) { + size_t w = ratioW * j; + size_t wid = (w < inImgW - 1) ? 1 : 0; + real w1lambda = ratioW * j - w; + real w2lambda = 1 - w1lambda; + // calculate four position for bilinear interpolation + const real* inPos = &inData[k * inputW + h * inImgW + w]; + real* outPos = &outData[k * outputW + i * outImgW + j]; + for (size_t c = 0; c < numChannels; ++c) { // loop for channels + // bilinear interpolation + outPos[0] = + h2lambda * (w2lambda * inPos[0] + w1lambda * inPos[wid]) + + h1lambda * (w2lambda * inPos[hid * inImgW] + + w1lambda * inPos[hid * inImgW + wid]); + inPos += inPosOffset; + outPos += outPosOffset; + } + } + } + } + } +} + +void CpuMatrix::bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + CHECK(dynamic_cast(&out)); + + size_t inputW = getWidth(); + size_t inputH = getHeight(); + size_t outputW = out.getWidth(); + size_t batchSize = out.getHeight(); + size_t inPosOffset = inImgH * inImgW; + size_t outPosOffset = outImgH * outImgW; + (void)(inputH); + + real* inGrad = getData(); + const real* outGrad = out.getData(); + + if (inImgH == outImgH && inImgW == outImgW) { + this->add(const_cast(out)); + } else { + for (size_t k = 0; k < batchSize; ++k) { // loop for batches + for (size_t i = 0; i < outImgH; ++i) { // loop for images + size_t h = ratioH * i; + size_t hid = (h < inImgH - 1) ? 1 : 0; + real h1lambda = ratioH * i - h; + real h2lambda = 1 - h1lambda; + for (size_t j = 0; j < outImgW; ++j) { + size_t w = ratioW * j; + size_t wid = (w < inImgW - 1) ? 1 : 0; + real w1lambda = ratioW * j - w; + real w2lambda = 1 - w1lambda; + + real* inPos = &inGrad[k * inputW + h * inImgW + w]; + const real* outPos = &outGrad[k * outputW + i * outImgW + j]; + for (size_t c = 0; c < numChannels; ++c) { // loop for channels + inPos[0] += h2lambda * w2lambda * outPos[0]; + inPos[wid] += h2lambda * w1lambda * outPos[0]; + inPos[hid * inImgW] += h1lambda * w2lambda * outPos[0]; + inPos[hid * inImgW + wid] += h1lambda * w1lambda * outPos[0]; + inPos += inPosOffset; + outPos += outPosOffset; + } + } + } + } + } +} + //////////////////////////////////////////////////////////////// // functions executed via cpu // //////////////////////////////////////////////////////////////// diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index cfb30797fcf1b..6c3c4804d2fc6 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -195,6 +195,8 @@ class Matrix : public BaseMatrix { virtual void resetOne() { LOG(FATAL) << "Not implemented"; } + void setDiag(real value); + virtual void copyFrom(const Matrix& src) { LOG(FATAL) << "Not implemented"; } virtual void trimFrom(const CpuSparseMatrix& src) { @@ -253,7 +255,7 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "copy data from int vector only available on CpuMatrix."; } - virtual void copyByRowIndex(Matrix& b, IVector& rowIndex) { + virtual void copyByRowIndex(Matrix& b, const IVector& rowIndex) { LOG(FATAL) << "Not implemented"; } @@ -328,6 +330,21 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "Not implemented"; } + virtual MatrixPtr getInverse() { + LOG(FATAL) << "Not implemented"; + return nullptr; + } + + /** + * @brief inverse. + * + * if allocate matInv's memory outside, then set memAlloc as false; + * else set as true. + */ + virtual void inverse(MatrixPtr matInv, bool memAlloc) { + LOG(FATAL) << "Not implemented"; + } + public: /// Only set all variables to 0 or NULL but not free them. virtual void clear() { @@ -343,11 +360,35 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "Not implemented"; } + virtual void addSharedBias(Matrix& b, real scale) { + LOG(FATAL) << "Not implemented"; + } + + virtual void addBias(Matrix& b, real scale, bool sharedBias) { + if (!sharedBias) { + addBias(b, scale); + } else { + addSharedBias(b, scale); + } + } + /// add each sample from a to this. virtual void collectBias(Matrix& a, real scale) { LOG(FATAL) << "Not implemented"; } + virtual void collectSharedBias(Matrix& a, real scale) { + LOG(FATAL) << "Not implemented"; + } + + virtual void collectBias(Matrix& a, real scale, bool sharedBias) { + if (!sharedBias) { + collectBias(a, scale); + } else { + collectSharedBias(a, scale); + } + } + virtual void sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { LOG(FATAL) << "Not implemented"; @@ -493,16 +534,40 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "Not implemeted"; } + /** + * set the max of each column of this to mat + */ virtual void colMax(Matrix& max) { LOG(FATAL) << "not implemented"; } + /** + * @brief Get the top k elements of each column of this matrix. + * + * The row ids and values of these elements are stored in + * maxIds and max respectively. where k is the size of maxIds. + * And note that the top k elements are not sorted. + */ + virtual void colMax(IVector& maxIds, Matrix& maxVal) { + LOG(FATAL) << "not implemented"; + } + + virtual void maxoutForward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + LOG(FATAL) << "not implemented"; + } + + virtual void maxoutBackward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + LOG(FATAL) << "not implemented"; + } + virtual void rowMaxId(IVector& maxIds) { LOG(FATAL) << "Not implemented"; } /** * @brief Get the top k elements of each row of this matrix. * * The column ids and values of these elements are stored in - * maxIds and max respectively. Note that the top k - * elements are not sorted. + * maxIds and max respectively. where k is the size of maxIds. + * And note that the top k elements are not sorted. */ virtual void rowMax(IVector& maxIds, Matrix& max) { LOG(FATAL) << "Not implemented"; @@ -742,31 +807,37 @@ class Matrix : public BaseMatrix { */ virtual void maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, size_t channels, size_t sizeX, - int start_, size_t stride, size_t outputH, - size_t outputW) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } /// Pooling backward operation. virtual void maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, - real scaleOutput) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } /// Pooling forward operation, caculate the average of sizeX elements. virtual void avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW) { + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } virtual void avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } @@ -924,6 +995,26 @@ class Matrix : public BaseMatrix { virtual void paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { LOG(FATAL) << "Not implemented"; } + virtual void bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + LOG(FATAL) << "Not implemented"; + } + virtual void bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + LOG(FATAL) << "Not implemented"; + } }; inline std::ostream& operator<<(std::ostream& os, const Matrix& mat) { @@ -948,6 +1039,7 @@ class GpuMatrix : public Matrix { void zeroMem(); void resetOne(); + void setDiag(real value); void resize(size_t newHeight, size_t newWidth); void resize(size_t newHeight, size_t newWidth, @@ -973,7 +1065,7 @@ class GpuMatrix : public Matrix { void copyFrom(const IVector& src); - void copyByRowIndex(Matrix& b, IVector& rowIndex); + void copyByRowIndex(Matrix& b, const IVector& rowIndex); MatrixPtr clone(size_t height, size_t width, bool useGpu = false); @@ -989,8 +1081,12 @@ class GpuMatrix : public Matrix { MatrixPtr getTranspose(); void transpose(MatrixPtr matTrans, bool memAlloc); + MatrixPtr getInverse(); + void inverse(MatrixPtr matInv, bool memAlloc); + /// add b to each sample of this. void addBias(Matrix& b, real scale); + void addSharedBias(Matrix& b, real scale); /** * @code @@ -998,6 +1094,7 @@ class GpuMatrix : public Matrix { * @endcode */ void collectBias(Matrix& a, real scale); + void collectSharedBias(Matrix& a, real scale); void sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode); @@ -1079,6 +1176,9 @@ class GpuMatrix : public Matrix { void rowMax(Matrix& max); void rowMax(IVector& maxIds, Matrix& max); void colMax(Matrix& max); + void colMax(IVector& maxIds, Matrix& max); + void maxoutForward(Matrix& a, IVector& id, size_t channels, size_t groups); + void maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t groups); void oneHotCrossEntropy(Matrix& output, IVector& label); void oneHotCrossEntropyBp(Matrix& outputV, IVector& label); @@ -1131,21 +1231,30 @@ class GpuMatrix : public Matrix { real alpha = 1.0f, real beta = 0.0f); void maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start_, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, - Matrix& outGrad, Matrix& outV, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput); + Matrix& outGrad, Matrix& outV, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, real scaleOutput); + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, @@ -1176,6 +1285,28 @@ class GpuMatrix : public Matrix { int contextLength, int contextStart, int totalPad, size_t beginPad); + + void bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels, + const real ratioH, + const real ratioW); + + void bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels, + const real ratioH, + const real ratioW); + + void multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label); + + void multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label); }; class CpuMatrix : public Matrix { @@ -1195,6 +1326,8 @@ class CpuMatrix : public Matrix { void zeroMem(); void resetOne(); + void setDiag(real value); + void resize(size_t newHeight, size_t newWidth); void resize(size_t newHeight, size_t newWidth, size_t newNnz, /* used to allocate space */ @@ -1214,6 +1347,9 @@ class CpuMatrix : public Matrix { MatrixPtr getTranspose(); void transpose(MatrixPtr matTrans, bool memAlloc); + MatrixPtr getInverse(); + void inverse(MatrixPtr matInv, bool memAlloc); + void copyFrom(const Matrix& src); void copyFrom(const Matrix& src, hl_stream_t stream); @@ -1226,7 +1362,7 @@ class CpuMatrix : public Matrix { void copyFrom(CpuSparseMatrix& src); - void copyByRowIndex(Matrix& b, IVector& rowIndex); + void copyByRowIndex(Matrix& b, const IVector& rowIndex); MatrixPtr clone(size_t height, size_t width, bool useGpu = false); @@ -1242,21 +1378,31 @@ class CpuMatrix : public Matrix { real alpha = 1.0f, real beta = 0.0f); void maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start_, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, - Matrix& outGrad, Matrix& outV, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput); + Matrix& outGrad, Matrix& outV, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, real scaleOutput); + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, @@ -1289,9 +1435,11 @@ class CpuMatrix : public Matrix { public: /// add b to each sample of this. void addBias(Matrix& b, real scale); + void addSharedBias(Matrix& b, real scale); /// add each sample of a to this. void collectBias(Matrix& a, real scale); + void collectSharedBias(Matrix& a, real scale); void sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode); @@ -1370,6 +1518,9 @@ class CpuMatrix : public Matrix { void rowMax(Matrix& max); void rowMax(IVector& maxIds, Matrix& maxVal); void colMax(Matrix& max); + void colMax(IVector& maxIds, Matrix& maxVal); + void maxoutForward(Matrix& a, IVector& id, size_t channels, size_t groups); + void maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t groups); void rowNormalizeL1(Matrix& out); void oneHotCrossEntropy(Matrix& output, IVector& label); @@ -1444,6 +1595,24 @@ class CpuMatrix : public Matrix { void multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label); void multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label); void classificationErrorMulti(Matrix& output, Matrix& label, real threshold); + + void bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels, + const real ratioH, + const real ratioW); + + void bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels, + const real ratioH, + const real ratioW); }; class SharedCpuMatrix : public CpuMatrix { diff --git a/paddle/math/SparseRowMatrix.cpp b/paddle/math/SparseRowMatrix.cpp index 0b5de252258a9..6986624d25c7a 100644 --- a/paddle/math/SparseRowMatrix.cpp +++ b/paddle/math/SparseRowMatrix.cpp @@ -227,12 +227,18 @@ void CacheRowCpuMatrix::mul(CpuSparseMatrix* a, CpuMatrix* b, real scaleAB, void SparsePrefetchRowCpuMatrix::addRows(const unsigned int* ids, size_t len) { std::vector& localIndices = indexDictHandle_->localIndices; + for (size_t i = 0; i < len; i ++) { + CHECK_LT(*(ids + i), this->getHeight()) + << "id:" << *(ids + i) << "Height:" << this->getHeight() + << "sparse id value exceeds the max input dimension, " + << "it could be caused invalid input data samples"; + } localIndices.insert(localIndices.end(), ids, ids + len); } void SparsePrefetchRowCpuMatrix::addRows(MatrixPtr input) { CpuSparseMatrix* mat = dynamic_cast(input.get()); - CHECK(mat) << "only support non value sparse matrix"; + CHECK(mat) << "only support sparse matrix"; addRows(reinterpret_cast(mat->getCols()), mat->getElementCnt()); } @@ -243,7 +249,13 @@ void SparsePrefetchRowCpuMatrix::addRows(IVectorPtr ids) { int* index = ids->getData(); for (size_t i = 0; i < numSamples; ++i) { if (index[i] == -1) continue; - localIndices.push_back((unsigned int)index[i]); + + unsigned int id = (unsigned int)index[i]; + CHECK_LT(id, this->getHeight()) + << "id:" << id << "Height:" << this->getHeight() + << "sparse id value exceeds the max input dimension, " + << "it could be caused invalid input data samples"; + localIndices.push_back(id); } } diff --git a/paddle/math/Vector.cpp b/paddle/math/Vector.cpp index 7553ea25e09d2..23c9cacceaea2 100644 --- a/paddle/math/Vector.cpp +++ b/paddle/math/Vector.cpp @@ -21,6 +21,7 @@ limitations under the License. */ #include "paddle/utils/ThreadLocal.h" #include "paddle/utils/Thread.h" #include "paddle/utils/Flags.h" +#include "Matrix.h" #include "hl_gpu.h" #include "hl_table_apply.h" @@ -73,6 +74,31 @@ std::shared_ptr> VectorT::create(size_t size, } } +template <> +MatrixPtr VectorT::toOneHotSparseMatrix(size_t idRange, bool useGpu) { + LOG(FATAL) << "Wrong for real vector"; + return nullptr; +} + +template <> +MatrixPtr VectorT::toOneHotSparseMatrix(size_t idRange, bool useGpu) { + int height = getSize(); + int width = idRange; + MatrixPtr mat = Matrix::createSparseMatrix( + height, idRange, height, NO_VALUE, SPARSE_CSR, false, useGpu); + + CpuIVector cpuIds(height); + cpuIds.copyFrom(*this); + int *idData = cpuIds.getData(); + + for (int i = 0; i < height; i ++) { + const unsigned int id = idData[i]; + CHECK_LT(id, width); + mat->setRow(i, 1, &id, nullptr); + } + return mat; +} + template GpuVectorT::GpuVectorT(size_t size) : VectorT(size, std::make_shared(sizeof(T) * size), diff --git a/paddle/math/Vector.h b/paddle/math/Vector.h index ee0a83bf038f0..faf8186b6d10d 100644 --- a/paddle/math/Vector.h +++ b/paddle/math/Vector.h @@ -37,6 +37,8 @@ class BaseVector; class SyncThreadPool; +class Matrix; + template class BaseVector : public BaseMatrixT { public: @@ -155,6 +157,12 @@ class VectorT : public BaseVector { subVecFrom(src, interval.first, interval.second - interval.first); } + /** + * convert the vector to a sparse one_hot matrix of width idRange + * only applies to IVector + */ + std::shared_ptr toOneHotSparseMatrix(size_t idRange, bool useGpu); + /** * This function will crash if the size of src and dest is different. */ diff --git a/paddle/math/tests/CMakeLists.txt b/paddle/math/tests/CMakeLists.txt index eb72f11e1c653..247be983ba329 100644 --- a/paddle/math/tests/CMakeLists.txt +++ b/paddle/math/tests/CMakeLists.txt @@ -13,3 +13,4 @@ add_simple_unittest(test_sparseMatrixCompare) add_simple_unittest(test_perturbation) add_simple_unittest(test_CpuGpuVector) add_simple_unittest(test_Allocator) +add_simple_unittest(test_FPException) diff --git a/paddle/math/tests/test_FPException.cpp b/paddle/math/tests/test_FPException.cpp new file mode 100644 index 0000000000000..174278c2aaac4 --- /dev/null +++ b/paddle/math/tests/test_FPException.cpp @@ -0,0 +1,94 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +/** + * This test is about floating point calculation exception. + * Paddle catches FE_INVALID, FE DIVBYZERO and FE_OVERFLOW exceptions. + * + * Some exceptions occur in the middle of a set of formulas, + * that can be circumvented by some tricks. + * For example, + * calculate tanh + * b = 2.0 / (1.0 + exp(-2 * a)) - 1.0 + * + * If the result of (-2 * a) is too large, + * a FE_OVERFLOW exception occurs when calculating exp. + * But the result of tanh is no overflow problem, + * so we can add some tricks to prevent exp calculate an excessive value. + * + */ +#include +#include +#include "paddle/math/Matrix.h" +#include "paddle/utils/Excepts.h" + +using namespace paddle; // NOLINT + +void SetTensorValue(Matrix& matrix, real value) { + int height = matrix.getHeight(); + int width = matrix.getWidth(); + int stride = matrix.getStride(); + real* data = matrix.getData(); + for (int i = 0; i < height; i++) { + int j = rand() % width; // NOLINT + if (typeid(matrix) == typeid(CpuMatrix)) { + data[i * stride + j] = value; + } else if (typeid(matrix) == typeid(GpuMatrix)) { + hl_memcpy(&data[i * stride + j], &value, sizeof(real)); + } else { + LOG(FATAL) << "should not reach here"; + } + } +} + +template +void testTanh(real illegal) { + MatrixPtr A = std::make_shared(10, 10); + MatrixPtr B = std::make_shared(10, 10); + A->randomizeUniform(); + B->randomizeUniform(); + + SetTensorValue(*A, illegal); + + A->tanh(*B); +} + +template +void testSigmoid(real illegal) { + MatrixPtr A = std::make_shared(10, 10); + MatrixPtr B = std::make_shared(10, 10); + A->randomizeUniform(); + B->randomizeUniform(); + + SetTensorValue(*A, illegal); + + A->sigmoid(*B); +} + +TEST(fp, overflow) { + for (auto illegal : {-90.0, 90.0}) { + LOG(INFO) << " illegal=" << illegal; + testTanh(illegal); + testSigmoid(illegal); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + + feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW); + return RUN_ALL_TESTS(); +} diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index fe8eacc2efbc5..9c03695ba5055 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -21,6 +21,8 @@ limitations under the License. */ #include "paddle/math/SparseMatrix.h" #include #include "paddle/gserver/tests/TestUtil.h" +#include "paddle/utils/Stat.h" + using namespace paddle; // NOLINT using namespace std; // NOLINT @@ -88,6 +90,73 @@ void MatrixCheckErr(const Matrix& matrix1, const Matrix& matrix2) { EXPECT_EQ(count, 0) << "There are " << count << " different element."; } +void testBilinearFwdBwd(int numSamples, int imgSizeH, int imgSizeW, + int channels) { + int inWidth = imgSizeH * imgSizeW * channels; + int outWidth = 2 * imgSizeH * 2 * imgSizeW * channels; + real ratioH = 0.5; + real ratioW = 0.5; + // forward + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + + input->randomizeUniform(); + inputGpu->copyFrom(*input); + + target->bilinearForward(*input, imgSizeH, imgSizeW, + 2 * imgSizeH, 2 * imgSizeW, channels, ratioH, ratioW); + targetGpu->bilinearForward(*inputGpu, imgSizeH, imgSizeW, + 2 * imgSizeH, 2 * imgSizeW, channels, ratioH, ratioW); + + // check + targetCheck->copyFrom(*targetGpu); + MatrixCheckErr(*target, *targetCheck); + + // backward + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, false, + true); + MatrixPtr targetCheckGrad = + CpuMatrix::create(numSamples, inWidth, false, false); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->bilinearBackward(*targetGrad, 2 * imgSizeH, 2 * imgSizeW, + imgSizeH, imgSizeW, channels, ratioH, ratioW); + inputGpuGrad->bilinearBackward(*targetGpuGrad, 2 * imgSizeH, 2 * imgSizeW, + imgSizeH, imgSizeW, channels, ratioH, ratioW); + + // check + targetCheckGrad->copyFrom(*inputGpuGrad); + MatrixCheckErr(*inputGrad, *targetCheckGrad); +} + +TEST(Matrix, BilinearFwdBwd) { + for (auto numSamples : {5, 10}) { + for (auto channels : {8, 16}) { + for (auto imgSizeH : {14, 28}) { + for (auto imgSizeW : {16, 30}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels + << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW; + testBilinearFwdBwd(numSamples, imgSizeH, imgSizeW, channels); + } + } + } + } +} + void testMatrixProjectionForward(int contextStart, int contextLength, bool padding, int batchSize, int inputDim) { MatrixPtr cpuInput = std::make_shared(batchSize, inputDim); @@ -639,9 +708,35 @@ void testMatrixTranspose(int height, int width) { MatrixCheckEqual(*cpuT, *outputCheck); } +void testMatrixInverse(int height) { + MatrixPtr cpu = std::make_shared(height, height); + MatrixPtr gpu = std::make_shared(height, height); + MatrixPtr cpuI = std::make_shared(height, height); + MatrixPtr gpuI = std::make_shared(height, height); + + /* Make matrix well conditioned: cpu * cpuT + Identity */ + cpu->randomizeUniform(); + MatrixPtr cpuT = cpu->getTranspose(); + MatrixPtr outputCheck = std::make_shared(height, height); + outputCheck->mul(cpu, cpuT); + cpu->setDiag(1.0); + cpu->add(*outputCheck); + + gpu->copyFrom(*cpu); + cpu->inverse(cpuI, false); + gpu->inverse(gpuI, false); + + outputCheck->copyFrom(*gpuI); + MatrixCheckErr(*cpuI, *outputCheck); + + outputCheck->mul(cpu, cpuI); + cpu->setDiag(1.0); + MatrixCheckErr(*cpu, *outputCheck); +} + TEST(Matrix, unary) { - for (auto height : {1, 11, 73, 128, 200, 330}) { - for (auto width : {1, 32, 100, 512, 1000, 3210}) { + for (auto height : {1, 3, 11, 73, 128, 200, 330}) { + for (auto width : {1, 3, 32, 100, 512, 1000, 3210}) { VLOG(3) << " height=" << height << " width=" << width; // applyUnary @@ -673,6 +768,8 @@ TEST(Matrix, unary) { // transpose testMatrixTranspose(height, width); } + // inverse + testMatrixInverse(height); } } @@ -1846,6 +1943,338 @@ TEST(Matrix, classificationError) { } } +void testMaxPoolFwdBwd(int numSamples, int channels, + int imgSizeH, int imgSizeW, + int ksizeH, int ksizeW, + int strideH, int strideW, + int padH, int padW) { + int outH = 0, outW = 0; + outH = (imgSizeH - ksizeH + 2 * padH + strideH - 1) / strideH + 1; + outW = (imgSizeW - ksizeW + 2 * padW + strideW - 1) / strideW + 1; + + int inWidth = imgSizeH * imgSizeW * channels; + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + int outWidth = channels * outH * outW; + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + + input->randomizeUniform(); + target->randomizeUniform(); + inputGpu->copyFrom(*input); + targetGpu->copyFrom(*target); + + target->maxPoolForward(*input, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + targetGpu->maxPoolForward(*inputGpu, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + targetCheck->copyFrom(*targetGpu); + checkMatrixEqual(target, targetCheck); + + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, + false, true); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->maxPoolBackward(*input, imgSizeH, imgSizeW, + *targetGrad, *target, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + inputGpuGrad->maxPoolBackward(*inputGpu, imgSizeH, imgSizeW, + *targetGpuGrad, *targetGpu, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + MatrixPtr targetBwdCheck = CpuMatrix::create(numSamples, inWidth, + false, false); + targetBwdCheck->copyFrom(*inputGpuGrad); + checkMatrixEqual(inputGrad, targetBwdCheck); +} + +void testAvgPoolFwdBwd(int numSamples, int channels, + int imgSizeH, int imgSizeW, + int ksizeH, int ksizeW, + int strideH, int strideW, + int padH, int padW) { + int outH = 0, outW = 0; + outH = (imgSizeH - ksizeH + 2 * padH + strideH - 1) / strideH + 1; + outW = (imgSizeW - ksizeW + 2 * padW + strideW - 1) / strideW + 1; + + int inWidth = imgSizeH * imgSizeW * channels; + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + int outWidth = channels * outH * outW; + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + + input->randomizeUniform(); + target->randomizeUniform(); + inputGpu->copyFrom(*input); + targetGpu->copyFrom(*target); + + target->avgPoolForward(*input, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + targetGpu->avgPoolForward(*inputGpu, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + targetCheck->copyFrom(*targetGpu); + MatrixCheckErr(*target, *targetCheck); + + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, + false, true); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->avgPoolBackward(*targetGrad, imgSizeH, imgSizeW, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + inputGpuGrad->avgPoolBackward(*targetGpuGrad, imgSizeH, imgSizeW, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + MatrixPtr targetBwdCheck = CpuMatrix::create(numSamples, inWidth, + false, false); + targetBwdCheck->copyFrom(*inputGpuGrad); + MatrixCheckErr(*inputGrad, *targetBwdCheck); +} + +TEST(Matrix, PoolFwdBwd) { + for (auto numSamples : {5, 32}) { + for (auto channels : {1, 9, 32}) { + for (auto imgSizeH : {14, 28}) { + for (auto imgSizeW : {16, 30}) { + for (auto sizeX : {2, 5}) { + for (auto sizeY : {2, 5}) { + for (auto sH : {1, 2}) { + for (auto sW : {1, 2}) { + for (auto pH : {0, (sizeY - 1)/2}) { + for (auto pW : {0, (sizeX - 1)/2}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels + << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW + << " sizeX=" << sizeX + << " sizeY=" << sizeY + << " strideH=" << sH + << " strideW=" << sW + << " padingH=" << pH + << " padingW=" << pW; + testMaxPoolFwdBwd(numSamples, channels, imgSizeH, + imgSizeW, sizeX, sizeY, sH, sW, pH, pW); + testAvgPoolFwdBwd(numSamples, channels, imgSizeH, + imgSizeW, sizeX, sizeY, sH, sW, pH, pW); + } + } + } + } + } + } + } + } + } + } +} + +void testMaxOutFwdBwd(int numSamples, int imgSizeH, int imgSizeW, + int channels, int groups) { + int inWidth = imgSizeH * imgSizeW * channels; + int outChannels = channels / groups; + int outWidth = imgSizeH * imgSizeW * outChannels; + + // forward + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + + IVectorPtr id = CpuIVector::create(numSamples * outWidth, false); + IVectorPtr idGpu = GpuIVector::create(numSamples * outWidth, true); + IVectorPtr idCheck = CpuIVector::create(numSamples * outWidth, false); + + input->randomizeUniform(); + inputGpu->copyFrom(*input); + + target->maxoutForward(*input, *id, outChannels, groups); + targetGpu->maxoutForward(*inputGpu, *idGpu, outChannels, groups); + + // check + targetCheck->copyFrom(*targetGpu); + MatrixCheckErr(*target, *targetCheck); + idCheck->copyFrom(*idGpu); + VectorCheckEqual(*id, *idCheck); + + // backward + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, false, + true); + MatrixPtr targetCheckGrad = CpuMatrix::create(numSamples, inWidth, false, + false); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->maxoutBackward(*targetGrad, *id, outChannels, groups); + inputGpuGrad->maxoutBackward(*targetGpuGrad, *idGpu, outChannels, groups); + + // check + targetCheckGrad->copyFrom(*inputGpuGrad); + MatrixCheckErr(*inputGrad, *targetCheckGrad); +} + +TEST(Matrix, MaxOutFwdBwd) { + for (auto numSamples : {5, 10}) { + for (auto channels : {8, 16}) { + for (auto imgSizeH : {14, 28}) { + for (auto imgSizeW : {16, 30}) { + for (auto groups : {2, 4}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels + << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW + << " groups=" << groups; + testMaxOutFwdBwd(numSamples, imgSizeH, imgSizeW, channels, groups); + } + } + } + } + } +} + +void testAddSharedBias(int numSamples, int dim, int channel) { + MatrixPtr cpuData = std::make_shared(numSamples, dim); + MatrixPtr gpuData = std::make_shared(numSamples, dim); + + MatrixPtr cpuBias = std::make_shared(1, channel); + MatrixPtr gpuBias = std::make_shared(1, channel); + + cpuData->randomizeUniform(); + gpuData->copyFrom(*cpuData); + cpuBias->randomizeUniform(); + gpuBias->copyFrom(*cpuBias); + + cpuData->addSharedBias(*cpuBias, 1.0); + gpuData->addSharedBias(*gpuBias, 1.0); + + MatrixPtr check = std::make_shared(numSamples, dim); + check->copyFrom(*gpuData); + MatrixCheckErr(*cpuData, *check); +} + +void testCollectSharedBias(int numSamples, int dim, int channel) { + MatrixPtr cpuData = std::make_shared(numSamples, dim); + MatrixPtr gpuData = std::make_shared(numSamples, dim); + + MatrixPtr cpuBias = std::make_shared(1, channel); + MatrixPtr gpuBias = std::make_shared(1, channel); + + cpuData->randomizeUniform(); + gpuData->copyFrom(*cpuData); + cpuBias->randomizeUniform(); + gpuBias->copyFrom(*cpuBias); + + cpuBias->collectSharedBias(*cpuData, 1.0); + gpuBias->collectSharedBias(*gpuData, 1.0); + + MatrixPtr check = std::make_shared(1, channel); + check->copyFrom(*gpuBias); + MatrixCheckErr(*cpuBias, *check); +} + +TEST(Matrix, sharedBias) { + for (auto numSamples : {1, 100, 520}) { + for (auto dim : {100 * 16, 100 * 32}) { + for (auto channel : {8, 16}) { + VLOG(3) << " numSamples=" << numSamples << " dim=" << dim + << " channel=" << channel; + testAddSharedBias(numSamples, dim, channel); + testCollectSharedBias(numSamples, dim, channel); + } + } + } +} + +void testMultiBinaryLabelCrossEntropy(int numSamples, int dim) { + MatrixPtr output = std::make_shared(numSamples, dim); + MatrixPtr cpuOutput = std::make_shared(numSamples, dim); + MatrixPtr gpuOutput = std::make_shared(numSamples, dim); + + MatrixPtr cpuEntropy = std::make_shared(numSamples, 1); + MatrixPtr gpuEntropy = std::make_shared(numSamples, 1); + + MatrixPtr cpuGrad = std::make_shared(numSamples, dim); + MatrixPtr gpuGrad = std::make_shared(numSamples, dim); + + MatrixPtr cpuLabel = std::make_shared + (numSamples, dim, numSamples, NO_VALUE, SPARSE_CSR, false); + MatrixPtr gpuLabel = std::make_shared + (numSamples, dim, numSamples, NO_VALUE, SPARSE_CSR, false); + for (int i = 0; i < numSamples; i ++) { + const unsigned int id = rand() % dim; // NOLINT + cpuLabel->setRow(i, 1, &id, nullptr); + gpuLabel->setRow(i, 1, &id, nullptr); + } + + output->randomizeUniform(); + cpuOutput->zeroMem(); + output->softmax(*cpuOutput); + gpuOutput->copyFrom(*cpuOutput); + + cpuEntropy->zeroMem(); + gpuEntropy->zeroMem(); + cpuEntropy->multiBinaryLabelCrossEntropy(*cpuOutput, *cpuLabel); + gpuEntropy->multiBinaryLabelCrossEntropy(*gpuOutput, *gpuLabel); + + MatrixPtr check1 = std::make_shared(numSamples, 1); + check1->copyFrom(*gpuEntropy); + MatrixCheckErr(*cpuEntropy, *check1); + + cpuGrad->zeroMem(); + gpuGrad->zeroMem(); + cpuGrad->multiBinaryLabelCrossEntropyBp(*cpuOutput, *cpuLabel); + gpuGrad->multiBinaryLabelCrossEntropyBp(*gpuOutput, *gpuLabel); + + MatrixPtr check2 = std::make_shared(numSamples, dim); + check2->copyFrom(*gpuGrad); + MatrixCheckErr(*cpuGrad, *check2); +} + +TEST(Matrix, multiBinaryCrossEntropy) { + for (auto numSamples : {100, 1000, 10000}) { + for (auto dim : {100, 1000, 10000}) { + VLOG(3) << " numSamples=" << numSamples << " dim=" << dim; + testMultiBinaryLabelCrossEntropy(numSamples, dim); + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/parameter/CMakeLists.txt b/paddle/parameter/CMakeLists.txt index d6f67604c0348..a35e46997fb04 100644 --- a/paddle/parameter/CMakeLists.txt +++ b/paddle/parameter/CMakeLists.txt @@ -10,4 +10,4 @@ add_style_check_target(paddle_parameter ${PARAMETERS_HEADERS}) add_dependencies(paddle_parameter gen_proto_cpp) if(WITH_TESTING) add_subdirectory(tests) -endif() \ No newline at end of file +endif() diff --git a/paddle/parameter/Parameter.h b/paddle/parameter/Parameter.h index 2f9606dc68026..ff251fe89f9f8 100644 --- a/paddle/parameter/Parameter.h +++ b/paddle/parameter/Parameter.h @@ -146,6 +146,12 @@ class Parameter { } } + void enableBufType(ParameterType type) { + if (bufs_[type]) return; + bufs_[type] = Vector::createParallelVector(config_.size(), useGpu_); + bufs_[type]->zeroMem(); + } + void enableIntType(ParameterType type, size_t intStoreSize = 0) { if (!intBufs_[type]) { SetDevice device(deviceId_); diff --git a/paddle/parameter/tests/CMakeLists.txt b/paddle/parameter/tests/CMakeLists.txt index 177fb2fdfc045..cab264db8e500 100644 --- a/paddle/parameter/tests/CMakeLists.txt +++ b/paddle/parameter/tests/CMakeLists.txt @@ -1 +1 @@ -add_simple_unittest(test_common) \ No newline at end of file +add_simple_unittest(test_common) diff --git a/paddle/pserver/ParameterServer2.cpp b/paddle/pserver/ParameterServer2.cpp index 8f72c1988d167..c8f37d0bf4f84 100644 --- a/paddle/pserver/ParameterServer2.cpp +++ b/paddle/pserver/ParameterServer2.cpp @@ -264,6 +264,15 @@ void ParameterServer2::setParameter(const SendParameterRequest& request, std::vector blockIds; blockIds.reserve(request.blocks_size()); int bufferIndex = 0; + + if (!request.blocks().size()) { + LOG(WARNING) + << "--ports_num or --ports_num_for_sparse might be too large, " + << "or total dense parameter size or sparse parameters size " + << "might be too small, this psever doesn't store any parameter."; + return; + } + for (const auto& block : request.blocks()) { /// block size for parameter(e.g. 128 for sparse row, 1K for dense) uint64_t blockSize = getParameterConfig(block).parameter_block_size(); diff --git a/paddle/pserver/PserverForPython.h b/paddle/pserver/PserverForPython.h deleted file mode 100644 index 5bbeae8bd8b97..0000000000000 --- a/paddle/pserver/PserverForPython.h +++ /dev/null @@ -1,116 +0,0 @@ -/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once -#include "paddle/pserver/ParameterClient.h" -#include "paddle/pserver/ParameterServer.h" -#include "paddle/parameter/Parameter.h" -#include - -namespace paddle { - -struct PyObjectDeleter { - void operator()(PyObject* obj) { - if (obj) { - Py_DECREF(obj); - } - } -}; - -class ParameterClientPy : public ParameterClient { -protected: - typedef std::unique_ptr PyObjectPtr; - - std::vector parameter_; - int initArgc_; - char** initArgv_; - -public: - ParameterClientPy(std::vector configs, int argc, - std::vector argv, bool useGpu) { - initArgc_ = argc; - initArgv_ = new char* [argc]; - for (int i = 0; i < argc; i++) { - initArgv_[i] = new char[argv[i].size()]; - strcpy(initArgv_[i], // NOLINT - argv[i].c_str()); // NOLINT TODO(yuyang18): use snprintf instead. - } - ParameterConfig pyConfig; - ParameterPtr param; - for (auto& config : configs) { - pyConfig.ParseFromString(config); - param.reset(new Parameter(pyConfig, useGpu)); - parameter_.push_back(param); - } - Py_Initialize(); - CHECK(Py_IsInitialized()); - } - - ~ParameterClientPy() { - delete initArgv_; - Py_Finalize(); - } - - Parameter getParameter(int idx) { return *(parameter_[idx].get()); } - - void initClientPy() { - initMain(initArgc_, initArgv_); - CHECK(init(parameter_)) << "Init Client Failed."; - } - - void setConfigPy(std::string config) { - OptimizationConfig optConfig; - optConfig.ParseFromString(config); - setConfig(optConfig); - } - - bool inStatusPy(int status) { return inStatus(PServerStatus(status)); } - - void setStatusPy(int status) { setStatus(PServerStatus(status)); } - - void waitForStatusPy(int status) { waitForStatus(PServerStatus(status)); } - - void sendParameterPy(int updateMode, int parameterType, int numSamples, - real cost, bool sendBackParameter) { - sendParameter(ParameterUpdateMode(updateMode), ParameterType(parameterType), - int64_t(numSamples), real(cost), sendBackParameter); - } - - template - std::string asyncCallPy(const char* serviceName, const char* funcName, - const std::string in) { - ProtoIn protoIn; - ProtoOut protoOut; - std::mutex waitLock; - std::string data; - protoIn.ParseFromString(in); - waitLock.lock(); - auto callback = [&](ProtoOut* pOut, bool isSuccessful) { - if (isSuccessful) { - pOut->SerializeToString(&data); - } else { - LOG(INFO) << "Async Talk Failed."; - } - waitLock.unlock(); - }; - - ubClient_.asyncCall(serviceName, funcName, protoIn, - &protoOut, callback); - waitLock.lock(); - protoOut.SerializeToString(&data); - return data; - } -}; - -} // namespace paddle diff --git a/paddle/pserver/SocketChannel.cpp b/paddle/pserver/SocketChannel.cpp index b9d542a296ddd..20295d7cdc22b 100644 --- a/paddle/pserver/SocketChannel.cpp +++ b/paddle/pserver/SocketChannel.cpp @@ -157,7 +157,8 @@ void SocketChannel::writeMessage(const std::vector& userIovs) { std::vector iovs; iovs.reserve(userIovs.size() + 2); iovs.push_back({&header, sizeof(header)}); - iovs.push_back({&iovLengths[0], sizeof(iovLengths[0]) * header.numIovs}); + iovs.push_back({&iovLengths[0], static_cast( + sizeof(iovLengths[0]) * header.numIovs)}); iovs.insert(iovs.end(), userIovs.begin(), userIovs.end()); header.totalLength = 0; diff --git a/paddle/py_paddle/__init__.py b/paddle/py_paddle/__init__.py index f372068942ea3..f8399f9c63d81 100644 --- a/paddle/py_paddle/__init__.py +++ b/paddle/py_paddle/__init__.py @@ -15,9 +15,10 @@ from util import DataProviderWrapperConverter from dataprovider_converter import DataProviderConverter -__all__ = ['paddle', - 'DataProviderConverter', - 'DataProviderWrapperConverter', # for deprecated usage. - 'loadParameterFile'] +__all__ = [ + 'paddle', + 'DataProviderConverter', + 'DataProviderWrapperConverter', # for deprecated usage. + 'loadParameterFile' +] util.monkeypatches() - diff --git a/paddle/py_paddle/dataprovider_converter.py b/paddle/py_paddle/dataprovider_converter.py index 0366bb636c704..d64c7b20cb65a 100644 --- a/paddle/py_paddle/dataprovider_converter.py +++ b/paddle/py_paddle/dataprovider_converter.py @@ -45,10 +45,8 @@ def scan(self, dat): def finish_scan(self, argument): assert isinstance(argument, swig_paddle.Arguments) assert isinstance(self.input_type, dp2.InputType) - m = swig_paddle.Matrix.createDense(self.__mat__, - self.__height__, - self.input_type.dim, - False) + m = swig_paddle.Matrix.createDense(self.__mat__, self.__height__, + self.input_type.dim, False) argument.setSlotValue(self.pos, m) @@ -63,7 +61,8 @@ def __init__(self, input_type, pos): def scan(self, dat): self.extend_cols(dat) - self.__rows__.append(len(dat)) + self.__rows__.append(len(self.__cols__)) + self.__height__ += 1 def extend_cols(self, dat): self.__cols__.extend(dat) @@ -140,8 +139,10 @@ def convert(self, dat, argument=None): assert isinstance(argument, swig_paddle.Arguments) argument.resize(len(self.input_types)) - scanners = [DataProviderConverter.create_scanner(i, each_type) - for i, each_type in enumerate(self.input_types)] + scanners = [ + DataProviderConverter.create_scanner(i, each_type) + for i, each_type in enumerate(self.input_types) + ] for each_sample in dat: for each_step, scanner in zip(each_sample, scanners): @@ -170,11 +171,14 @@ def create_scanner(i, each): assert retv is not None if each.seq_type == dp2.SequenceType.SUB_SEQUENCE: - retv = SequenceScanner(each, i, retv, lambda a, p, seq: - a.setSlotSubSequenceStartPositions(p, seq)) - - if each.seq_type in [dp2.SequenceType.SUB_SEQUENCE, - dp2.SequenceType.SEQUENCE]: - retv = SequenceScanner(each, i, retv, lambda a, p, seq: - a.setSlotSequenceStartPositions(p, seq)) + retv = SequenceScanner( + each, i, retv, + lambda a, p, seq: a.setSlotSubSequenceStartPositions(p, seq)) + + if each.seq_type in [ + dp2.SequenceType.SUB_SEQUENCE, dp2.SequenceType.SEQUENCE + ]: + retv = SequenceScanner( + each, i, retv, + lambda a, p, seq: a.setSlotSequenceStartPositions(p, seq)) return retv diff --git a/paddle/py_paddle/util.py b/paddle/py_paddle/util.py index e6cf2710ef523..e1f310580f95c 100644 --- a/paddle/py_paddle/util.py +++ b/paddle/py_paddle/util.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Some Useful method for py_paddle. """ @@ -80,6 +79,19 @@ def wrap(callback): return __ParameterCallbackWrapper__(callback).__disown__() +def __arguments_to_numpy__(i, arg): + assert isinstance(arg, swig_paddle.Arguments) + value = arg.getSlotValue(i) + ids = arg.getSlotIds(i) + if value is not None: + assert isinstance(value, swig_paddle.Matrix) + value = value.copyToNumpyMat() + if ids is not None: + assert isinstance(ids, swig_paddle.IVector) + ids = ids.copyToNumpyArray() + return {"value": value, "id": ids} + + def __monkeypatch_gradient_machine__(): """ Add some class methods to GradientMachine. @@ -88,21 +100,6 @@ def __monkeypatch_gradient_machine__(): swig_paddle.GradientMachine.loadFromConfigFile = \ staticmethod(loadGradientMachine) - def __arguments_to_numpy__(i, arg): - assert isinstance(arg, swig_paddle.Arguments) - value = arg.getSlotValue(i) - if value is not None: - assert isinstance(value, swig_paddle.Matrix) - value = value.copyToNumpyMat() - ids = arg.getSlotIds(i) - if ids is not None: - assert isinstance(ids, swig_paddle.IVector) - ids = ids.copyToNumpyArray() - return { - "value": value, - "id": ids - } - def __matrix_to_numpy__(m): if isinstance(m, swig_paddle.Matrix): return m.copyToNumpyMat() @@ -113,9 +110,11 @@ def __matrix_to_numpy__(m): def createFromConfigProto(protoObj, createMode=swig_paddle.CREATE_MODE_NORMAL, - paramTypes=[swig_paddle.PARAMETER_VALUE, - swig_paddle.PARAMETER_GRADIENT, - swig_paddle.PARAMETER_MOMENTUM]): + paramTypes=[ + swig_paddle.PARAMETER_VALUE, + swig_paddle.PARAMETER_GRADIENT, + swig_paddle.PARAMETER_MOMENTUM + ]): """ Create Gradient Machine From Proto object. :param protoObj: Model config @@ -126,7 +125,7 @@ def createFromConfigProto(protoObj, :type paramTypes: list of int :return: paddle.GradientMachine """ - assert isinstance(protoObj, paddle.proto.ModelConfig_pb2.ModelConfig) + assert isinstance(protoObj, paddle.proto.ModelConfig) return swig_paddle.GradientMachine.createByConfigProtoStr( protoObj.SerializeToString(), createMode, paramTypes) @@ -145,8 +144,10 @@ def forwardTest(self, inArgs): """ outArgs = swig_paddle.Arguments.createArguments(0) self.forward(inArgs, outArgs, swig_paddle.PASS_TEST) - return [__arguments_to_numpy__(i, outArgs) for i in xrange( - outArgs.getSlotNum())] + return [ + __arguments_to_numpy__(i, outArgs) + for i in xrange(outArgs.getSlotNum()) + ] swig_paddle.GradientMachine.forwardTest = forwardTest @@ -167,7 +168,10 @@ def backward(self, callback): swig_paddle.GradientMachine.__forwardBackward__ = \ swig_paddle.GradientMachine.forwardBackward - def forwardBackward(self, inArgs, outArgs, passType, + def forwardBackward(self, + inArgs, + outArgs, + passType, callback=swig_paddle.UpdateCallback()): """ GradientMachine forward backward. @@ -315,9 +319,8 @@ def append(self, other): self.cols += other def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, - self.dim, - len(self.cols), True) + mat = swig_paddle.Matrix.createSparse( + len(self.indices) - 1, self.dim, len(self.cols), True) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols) self.putIntoArg(slot_idx, arg, mat) @@ -341,9 +344,8 @@ def append(self, other): self.values += map(lambda x: x[1], other) def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, - self.dim, - len(self.cols), False) + mat = swig_paddle.Matrix.createSparse( + len(self.indices) - 1, self.dim, len(self.cols), False) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols, self.values) self.putIntoArg(slot_idx, arg, mat) @@ -352,8 +354,9 @@ def __call__(self, slot_idx, arg): paddle.trainer.PyDataProviderWrapper.DenseSlot: DenseValueConverter, paddle.trainer.PyDataProviderWrapper.IndexSlot: IdValueConverter, paddle.trainer.PyDataProviderWrapper.SparseNonValueSlot: - SparseNonValueConverter, - paddle.trainer.PyDataProviderWrapper.SparseValueSlot: SparseValueConverter + SparseNonValueConverter, + paddle.trainer.PyDataProviderWrapper.SparseValueSlot: + SparseValueConverter } def __init__(self, use_seq, header): @@ -381,10 +384,9 @@ def convert(self, wrapper_data, argument=None): assert isinstance(argument, swig_paddle.Arguments) argument.resize(len(self.__header__)) - values = map(lambda x: - DataProviderWrapperConverter.__SLOT_VALUE_CONVERTER_MAP__[ - x.__class__](x), - self.__header__) + values = map( + lambda x: DataProviderWrapperConverter.__SLOT_VALUE_CONVERTER_MAP__[x.__class__](x), + self.__header__) if self.__use_seq__: seq_dim = [[] for _ in xrange(self.__header__.__len__())] @@ -394,14 +396,13 @@ def convert(self, wrapper_data, argument=None): for slot_idx, sequence in enumerate(each_sample): for raw_data in sequence: values[slot_idx].append(raw_data) - seq_start_pos[slot_idx].append( - seq_start_pos[slot_idx][-1] + len(sequence)) + seq_start_pos[slot_idx].append(seq_start_pos[slot_idx][-1] + + len(sequence)) seq_dim[slot_idx].append(len(sequence)) for slot_idx in xrange(len(self.__header__)): - argument.setSlotSequenceDim(slot_idx, - swig_paddle.IVector.create( - seq_dim[slot_idx])) + argument.setSlotSequenceDim( + slot_idx, swig_paddle.IVector.create(seq_dim[slot_idx])) argument.setSlotSequenceStartPositions( slot_idx, swig_paddle.IVector.create(seq_start_pos[slot_idx])) @@ -422,7 +423,6 @@ def __call__(self, wrapper_data, argument=None): return self.convert(wrapper_data, argument) - def __monkey_patch_protobuf_objects__(): def ParameterConfig_toProto(self): """ @@ -459,14 +459,28 @@ def OptimizationConfig_createFromProto(protoObj): :return: paddle.OptimizationConfig """ - assert isinstance(protoObj, - paddle.proto.TrainerConfig_pb2.OptimizationConfig) + assert isinstance(protoObj, paddle.proto.OptimizationConfig) return swig_paddle.OptimizationConfig.createFromProtoString( protoObj.SerializeToString()) swig_paddle.OptimizationConfig.createFromProto = staticmethod( OptimizationConfig_createFromProto) + def TrainerConfig_createFromProto(protoObj): + """ + Create a new paddle.TrainerConfig from + proto.OptimizationConfig + + :param protoObj: proto.TrainerConfig + :return: paddle.TrainerConfig + """ + assert isinstance(protoObj, paddle.proto.TrainerConfig) + return swig_paddle.TrainerConfig.createFromProtoString( + protoObj.SerializeToString()) + + swig_paddle.TrainerConfig.createFromProto = staticmethod( + TrainerConfig_createFromProto) + def __monkey_patch_parameter__(): def getBufs(self): @@ -483,9 +497,72 @@ def getBufs(self): swig_paddle.Parameter.getBufs = getBufs +def __monkey_patch_trainer__(): + swig_paddle.Trainer.__create__ = staticmethod(swig_paddle.Trainer.create) + + def Trainer_create(config, model=None): + """ + Create a trainer for model with TrainerCOnfig trainer_config + trainer_config.model_config will be ignored when model is supplied. + Trainer.trainOneBatch() and Trainer.forwardOneBatch() can be used only + when trainer_config.data_config is set. + + A typical usage for Trainer is: + .. code-block:: python + trainer = Trainer.create(trainer_config, model) + for p in xrange(num_passes) + while True: + data = get_next_batch(batch_size) + if not data: + break + trainer.trainOneDataBatch(batch_size, data) + trainer.finishTrainPass() + trainer.finishTrain() + + The trainer will take care of logging, model saving, distributed + training, etc. + + :param config: trainer configuration + :type config: paddle.proto.TrainerConfig + :param model: the model to be trained + :type model: swig_paddle.GradientMachine + :return: a trainer + :rtype swig_paddle.Trainer + + """ + assert isinstance(config, paddle.proto.TrainerConfig) + if model is not None: + assert isinstance(model, swig_paddle.GradientMachine) + return swig_paddle.Trainer.__create__( + swig_paddle.TrainerConfig.createFromProto(config), model) + + swig_paddle.Trainer.create = staticmethod(Trainer_create) + + swig_paddle.Trainer.__getForwardOutput__ = \ + swig_paddle.Trainer.getForwardOutput + + def getForwardOutput(self): + """ + Get the netword outputs from the previous trainOneBatch(), + trainOneDataBatch(), testOneDataPatch(), or forwardOneBatch() call. + + :return: list of dictionary with keys ['id', 'value'], each value is a + numpy.ndarray. + """ + outArgs = self.__getForwardOutput__() + return [ + __arguments_to_numpy__(i, outArgs) + for i in xrange(outArgs.getSlotNum()) + ] + + swig_paddle.Trainer.getForwardOutput = getForwardOutput + + def monkeypatches(): - patches = [__monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, - __monkey_patch_protobuf_objects__, - __monkey_patch_parameter__] + patches = [ + __monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, + __monkey_patch_protobuf_objects__, __monkey_patch_parameter__, + __monkey_patch_trainer__ + ] for patch in patches: patch() diff --git a/paddle/scripts/CMakeLists.txt b/paddle/scripts/CMakeLists.txt index dee46055c5a4d..1bae396a18688 100644 --- a/paddle/scripts/CMakeLists.txt +++ b/paddle/scripts/CMakeLists.txt @@ -6,4 +6,4 @@ configure_file(submit_local.sh.in install(FILES ${CMAKE_CURRENT_BINARY_DIR}/submit_local.sh DESTINATION bin PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ - RENAME paddle) \ No newline at end of file + RENAME paddle) diff --git a/paddle/scripts/cluster_train/conf.py b/paddle/scripts/cluster_train/conf.py index c8fd360e7552e..f1114a59201b9 100644 --- a/paddle/scripts/cluster_train/conf.py +++ b/paddle/scripts/cluster_train/conf.py @@ -13,17 +13,14 @@ # limitations under the License. HOSTS = [ - "root@192.168.100.17", - "root@192.168.100.18", - ] - + "root@192.168.100.17", + "root@192.168.100.18", +] ''' workspace configuration ''' #root dir for workspace, can be set as any director with real user account ROOT_DIR = "/home/paddle" - - ''' network configuration ''' @@ -37,4 +34,4 @@ PADDLE_PORTS_NUM_FOR_SPARSE = 2 #environments setting for all processes in cluster job -LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/lib64" +LD_LIBRARY_PATH = "/usr/local/cuda/lib64:/usr/lib64" diff --git a/paddle/scripts/cluster_train/paddle.py b/paddle/scripts/cluster_train/paddle.py index 79698c72e619f..7343a600c1bf5 100644 --- a/paddle/scripts/cluster_train/paddle.py +++ b/paddle/scripts/cluster_train/paddle.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - """ module for launching cluster job """ import os @@ -23,13 +21,13 @@ import time import signal - from fabric.api import run, put, settings, env, prefix from fabric.tasks import execute #configuration for cluster import conf + def refine_unknown_args(cmd_args): ''' refine unknown parameters to handle some special parameters @@ -37,7 +35,7 @@ def refine_unknown_args(cmd_args): new_args = [] for arg in cmd_args: if arg.startswith("--") and arg.find("=") != -1: - equal_pos = arg.find("=") #find first = pos + equal_pos = arg.find("=") #find first = pos arglist = list(arg) arglist[equal_pos] = " " arg = "".join(arglist) @@ -50,6 +48,7 @@ def refine_unknown_args(cmd_args): new_args.append(arg) return new_args + def kill_process(): ''' kill comments threads @@ -60,6 +59,7 @@ def kill_process(): | awk '{print $2}' \ | xargs kill > /dev/null 2>&1") + def job_prepare(jobdir, data=None): ''' prepare job related workspace data @@ -70,6 +70,7 @@ def job_prepare(jobdir, data=None): This function just prepare all related model and other resources needed at runtime. ''' + def job_create_workspace(jobdir, data=None): ''' prepare job workspace, common file, etc. @@ -94,7 +95,8 @@ def set_nodefile(nodeid): execute(set_nodefile, i, hosts=conf.HOSTS[i]) #clean rubbish caused by exception with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) + execute(kill_process, hosts=conf.HOSTS) + def job_pserver(jobdir, pids=None): ''' @@ -124,9 +126,8 @@ def start_pserver(jobdir, pargs): execute(start_pserver, jobdir, pargs, hosts=conf.HOSTS) -def job_trainer(jobdir, - train_args_dict, - pids=None): + +def job_trainer(jobdir, train_args_dict, pids=None): ''' start paddle trainer ''' @@ -171,9 +172,8 @@ def start_trainer(jobdir, args): train_args += " --trainer_id=" + str(i) execute(start_trainer, jobdir, train_args, hosts=conf.HOSTS[i]) -def job_all(job_package, - jobdir=None, - train_args_dict=None): + +def job_all(job_package, jobdir=None, train_args_dict=None): ''' param job_package param train_args_dict @@ -183,41 +183,52 @@ def job_all(job_package, jobdir = conf.ROOT_DIR + "/JOB" + timestamp job_prepare(jobdir, job_package) job_pserver(jobdir) - time.sleep(5) #wait until pservers completely start + time.sleep(5) #wait until pservers completely start job_trainer(jobdir, train_args_dict) job_clean() + def job_clean(): ''' if starting job failed from paddle internal, the framework always is launched successfully since these process are daemon processes. so this job_clean can alway clean job rubbish process with ctrl+c. ''' + def signal_handler(signal, frame): ''' SIGINT handler ''' + def kill_process(): - run("ps aux \ + run("ps aux \ | grep paddle_process_by_paddle \ | grep -v grep \ | awk '{print $2}' \ | xargs kill > /dev/null 2>&1") + with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) + execute(kill_process, hosts=conf.HOSTS) signal.signal(signal.SIGINT, signal_handler) signal.pause() + if __name__ == '__main__': - parser = argparse.ArgumentParser(prog="paddle.py", - description='simple tool for cluster training') - parser.add_argument('-j', '--job_workspace', - required=False, default=None, - help='job workspace') - parser.add_argument('-p', '--job_dispatch_package', - required=False, default=None, - help='job package for dispatching to all other nodes') + parser = argparse.ArgumentParser( + prog="paddle.py", description='simple tool for cluster training') + parser.add_argument( + '-j', + '--job_workspace', + required=False, + default=None, + help='job workspace') + parser.add_argument( + '-p', + '--job_dispatch_package', + required=False, + default=None, + help='job package for dispatching to all other nodes') args, train_args_list = parser.parse_known_args() train_args = refine_unknown_args(train_args_list) @@ -227,14 +238,10 @@ def kill_process(): #if assigned workspace, do not need to dispatch data, #so job_local_package should be None assert args.job_dispatch_package is None - job_all(None, - args.job_workspace, - train_args_dict) + job_all(None, args.job_workspace, train_args_dict) elif args.job_dispatch_package is not None: assert args.job_workspace is None assert os.path.isdir(args.job_dispatch_package) - job_all(args.job_dispatch_package, - None, - train_args_dict) + job_all(args.job_dispatch_package, None, train_args_dict) else: print "--job_workspace or --job_dispatch_package should be set" diff --git a/paddle/scripts/cpplint.py b/paddle/scripts/cpplint.py index 5e905b865fc51..157ce7b44ac3c 100644 --- a/paddle/scripts/cpplint.py +++ b/paddle/scripts/cpplint.py @@ -27,7 +27,6 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """Does google-lint on c++ files. The goal of this script is to identify places in the code that *may* @@ -55,7 +54,6 @@ import sys import unicodedata - _USAGE = """ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] [--counting=total|toplevel|detailed] [--root=subdir] @@ -242,13 +240,11 @@ 'whitespace/semicolon', 'whitespace/tab', 'whitespace/todo', - ] +] # These error categories are no longer enforced by cpplint, but for backwards- # compatibility they may still appear in NOLINT comments. -_LEGACY_ERROR_CATEGORIES = [ - 'readability/streams', - ] +_LEGACY_ERROR_CATEGORIES = ['readability/streams', ] # The default state of the category filter. This is overridden by the --filter= # flag. By default all errors are on, so only add here categories that should be @@ -394,8 +390,7 @@ 'cuchar', 'cwchar', 'cwctype', - ]) - +]) # These headers are excluded from [build/include] and [build/include_order] # checks: @@ -405,38 +400,40 @@ _THIRD_PARTY_HEADERS_PATTERN = re.compile( r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$') - # Assertion macros. These are defined in base/logging.h and # testing/base/gunit.h. Note that the _M versions need to come first # for substring matching to work. _CHECK_MACROS = [ - 'DCHECK', 'CHECK', - 'EXPECT_TRUE_M', 'EXPECT_TRUE', - 'ASSERT_TRUE_M', 'ASSERT_TRUE', - 'EXPECT_FALSE_M', 'EXPECT_FALSE', - 'ASSERT_FALSE_M', 'ASSERT_FALSE', - ] + 'DCHECK', + 'CHECK', + 'EXPECT_TRUE_M', + 'EXPECT_TRUE', + 'ASSERT_TRUE_M', + 'ASSERT_TRUE', + 'EXPECT_FALSE_M', + 'EXPECT_FALSE', + 'ASSERT_FALSE_M', + 'ASSERT_FALSE', +] # Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE _CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) -for op, replacement in [('==', 'EQ'), ('!=', 'NE'), - ('>=', 'GE'), ('>', 'GT'), +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), ('>=', 'GE'), ('>', 'GT'), ('<=', 'LE'), ('<', 'LT')]: - _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement - _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement - -for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), - ('>=', 'LT'), ('>', 'LE'), - ('<=', 'GT'), ('<', 'GE')]: - _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement - _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement + +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), ('>=', 'LT'), + ('>', 'LE'), ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement + _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement # Alternative tokens and their replacements. For full list, see section 2.5 # Alternative tokens [lex.digraph] in the C++ standard. @@ -455,16 +452,15 @@ 'xor_eq': '^=', 'not': '!', 'not_eq': '!=' - } +} # Compile regular expression that matches all the above keywords. The "[ =()]" # bit is meant to avoid matching these keywords outside of boolean expressions. # # False positives include C-style multi-line comments and multi-line strings # but those have always been troublesome for cpplint. -_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( - r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') - +_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile(r'[ =()](' + ('|'.join( + _ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') # These constants define types of headers for use with # _IncludeState.CheckNextIncludeOrder(). @@ -475,17 +471,16 @@ _OTHER_HEADER = 5 # These constants define the current inline assembly state -_NO_ASM = 0 # Outside of inline assembly block -_INSIDE_ASM = 1 # Inside inline assembly block -_END_ASM = 2 # Last line of inline assembly block -_BLOCK_ASM = 3 # The whole block is an inline assembly block +_NO_ASM = 0 # Outside of inline assembly block +_INSIDE_ASM = 1 # Inside inline assembly block +_END_ASM = 2 # Last line of inline assembly block +_BLOCK_ASM = 3 # The whole block is an inline assembly block # Match start of assembly blocks _MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' r'(?:\s+(volatile|__volatile__))?' r'\s*[{(]') - _regexp_compile_cache = {} # {str, set(int)}: a map from error categories to sets of linenumbers @@ -504,8 +499,9 @@ # This is set by --extensions flag. _valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh']) + def ParseNolintSuppressions(filename, raw_line, linenum, error): - """Updates the global list of error-suppressions. + """Updates the global list of error-suppressions. Parses any NOLINT comments on the current line, updating the global error_suppressions store. Reports an error if the NOLINT comment @@ -517,45 +513,47 @@ def ParseNolintSuppressions(filename, raw_line, linenum, error): linenum: int, the number of the current line. error: function, an error handler. """ - matched = Search(r'\bNOLINT(NEXTLINE(S_\d+)?)?\b(\([^)]+\))?', raw_line) - if matched: - if matched.group(1): - lines = matched.group(2) - if lines : - lines=int(lines[2:]) - suppressed_line = [ linenum + i for i in xrange(lines) ] - else: - suppressed_line = linenum + 1 - else: - suppressed_line = linenum - category = matched.group(3) - if category in (None, '(*)'): # => "suppress all" - if isinstance(suppressed_line, int): - _error_suppressions.setdefault(None, set()).add(suppressed_line) - else: - for _line in suppressed_line: - _error_suppressions.setdefault(None, set()).add(_line) - else: - if category.startswith('(') and category.endswith(')'): - category = category[1:-1] - if category in _ERROR_CATEGORIES: - if isinstance(suppressed_line, int): - _error_suppressions.setdefault(category, set()).add(suppressed_line) - else: - for _line in suppressed_line: - _error_suppressions.setdefault(category, set()).add(_line) - elif category not in _LEGACY_ERROR_CATEGORIES: - error(filename, linenum, 'readability/nolint', 5, - 'Unknown NOLINT error category: %s' % category) + matched = Search(r'\bNOLINT(NEXTLINE(S_\d+)?)?\b(\([^)]+\))?', raw_line) + if matched: + if matched.group(1): + lines = matched.group(2) + if lines: + lines = int(lines[2:]) + suppressed_line = [linenum + i for i in xrange(lines)] + else: + suppressed_line = linenum + 1 + else: + suppressed_line = linenum + category = matched.group(3) + if category in (None, '(*)'): # => "suppress all" + if isinstance(suppressed_line, int): + _error_suppressions.setdefault(None, set()).add(suppressed_line) + else: + for _line in suppressed_line: + _error_suppressions.setdefault(None, set()).add(_line) + else: + if category.startswith('(') and category.endswith(')'): + category = category[1:-1] + if category in _ERROR_CATEGORIES: + if isinstance(suppressed_line, int): + _error_suppressions.setdefault( + category, set()).add(suppressed_line) + else: + for _line in suppressed_line: + _error_suppressions.setdefault(category, + set()).add(_line) + elif category not in _LEGACY_ERROR_CATEGORIES: + error(filename, linenum, 'readability/nolint', 5, + 'Unknown NOLINT error category: %s' % category) def ResetNolintSuppressions(): - """Resets the set of NOLINT suppressions to empty.""" - _error_suppressions.clear() + """Resets the set of NOLINT suppressions to empty.""" + _error_suppressions.clear() def IsErrorSuppressedByNolint(category, linenum): - """Returns true if the specified error category is suppressed on this line. + """Returns true if the specified error category is suppressed on this line. Consults the global error_suppressions map populated by ParseNolintSuppressions/ResetNolintSuppressions. @@ -566,22 +564,22 @@ def IsErrorSuppressedByNolint(category, linenum): Returns: bool, True iff the error should be suppressed due to a NOLINT comment. """ - return (linenum in _error_suppressions.get(category, set()) or - linenum in _error_suppressions.get(None, set())) + return (linenum in _error_suppressions.get(category, set()) or + linenum in _error_suppressions.get(None, set())) def Match(pattern, s): - """Matches the string with the pattern, caching the compiled regexp.""" - # The regexp compilation caching is inlined in both Match and Search for - # performance reasons; factoring it out into a separate function turns out - # to be noticeably expensive. - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].match(s) + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both Match and Search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) def ReplaceAll(pattern, rep, s): - """Replaces instances of pattern in a string with a replacement. + """Replaces instances of pattern in a string with a replacement. The compiled regex is kept in a cache shared by Match and Search. @@ -593,20 +591,20 @@ def ReplaceAll(pattern, rep, s): Returns: string with replacements made (or original string if no replacements) """ - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].sub(rep, s) + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].sub(rep, s) def Search(pattern, s): - """Searches the string for the pattern, caching the compiled regexp.""" - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].search(s) + """Searches the string for the pattern, caching the compiled regexp.""" + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) class _IncludeState(object): - """Tracks line numbers for includes, and the order in which includes appear. + """Tracks line numbers for includes, and the order in which includes appear. include_list contains list of lists of (header, line number) pairs. It's a lists of lists rather than just one flat list to make it @@ -617,35 +615,35 @@ class _IncludeState(object): raise an _IncludeError with an appropriate error message. """ - # self._section will move monotonically through this set. If it ever - # needs to move backwards, CheckNextIncludeOrder will raise an error. - _INITIAL_SECTION = 0 - _MY_H_SECTION = 1 - _C_SECTION = 2 - _CPP_SECTION = 3 - _OTHER_H_SECTION = 4 - - _TYPE_NAMES = { - _C_SYS_HEADER: 'C system header', - _CPP_SYS_HEADER: 'C++ system header', - _LIKELY_MY_HEADER: 'header this file implements', - _POSSIBLE_MY_HEADER: 'header this file may implement', - _OTHER_HEADER: 'other header', - } - _SECTION_NAMES = { - _INITIAL_SECTION: "... nothing. (This can't be an error.)", - _MY_H_SECTION: 'a header this file implements', - _C_SECTION: 'C system header', - _CPP_SECTION: 'C++ system header', - _OTHER_H_SECTION: 'other header', - } - - def __init__(self): - self.include_list = [[]] - self.ResetSection('') - - def FindHeader(self, header): - """Check if a header has already been included. + # self._section will move monotonically through this set. If it ever + # needs to move backwards, CheckNextIncludeOrder will raise an error. + _INITIAL_SECTION = 0 + _MY_H_SECTION = 1 + _C_SECTION = 2 + _CPP_SECTION = 3 + _OTHER_H_SECTION = 4 + + _TYPE_NAMES = { + _C_SYS_HEADER: 'C system header', + _CPP_SYS_HEADER: 'C++ system header', + _LIKELY_MY_HEADER: 'header this file implements', + _POSSIBLE_MY_HEADER: 'header this file may implement', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing. (This can't be an error.)", + _MY_H_SECTION: 'a header this file implements', + _C_SECTION: 'C system header', + _CPP_SECTION: 'C++ system header', + _OTHER_H_SECTION: 'other header', + } + + def __init__(self): + self.include_list = [[]] + self.ResetSection('') + + def FindHeader(self, header): + """Check if a header has already been included. Args: header: header to check. @@ -653,35 +651,35 @@ def FindHeader(self, header): Line number of previous occurrence, or -1 if the header has not been seen before. """ - for section_list in self.include_list: - for f in section_list: - if f[0] == header: - return f[1] - return -1 + for section_list in self.include_list: + for f in section_list: + if f[0] == header: + return f[1] + return -1 - def ResetSection(self, directive): - """Reset section checking for preprocessor directive. + def ResetSection(self, directive): + """Reset section checking for preprocessor directive. Args: directive: preprocessor directive (e.g. "if", "else"). """ - # The name of the current section. - self._section = self._INITIAL_SECTION - # The path of last found header. - self._last_header = '' + # The name of the current section. + self._section = self._INITIAL_SECTION + # The path of last found header. + self._last_header = '' - # Update list of includes. Note that we never pop from the - # include list. - if directive in ('if', 'ifdef', 'ifndef'): - self.include_list.append([]) - elif directive in ('else', 'elif'): - self.include_list[-1] = [] + # Update list of includes. Note that we never pop from the + # include list. + if directive in ('if', 'ifdef', 'ifndef'): + self.include_list.append([]) + elif directive in ('else', 'elif'): + self.include_list[-1] = [] - def SetLastHeader(self, header_path): - self._last_header = header_path + def SetLastHeader(self, header_path): + self._last_header = header_path - def CanonicalizeAlphabeticalOrder(self, header_path): - """Returns a path canonicalized for alphabetical comparison. + def CanonicalizeAlphabeticalOrder(self, header_path): + """Returns a path canonicalized for alphabetical comparison. - replaces "-" with "_" so they both cmp the same. - removes '-inl' since we don't require them to be after the main header. @@ -693,10 +691,10 @@ def CanonicalizeAlphabeticalOrder(self, header_path): Returns: Canonicalized path. """ - return header_path.replace('-inl.h', '.h').replace('-', '_').lower() + return header_path.replace('-inl.h', '.h').replace('-', '_').lower() - def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): - """Check if a header is in alphabetical order with the previous header. + def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): + """Check if a header is in alphabetical order with the previous header. Args: clean_lines: A CleansedLines instance containing the file. @@ -706,18 +704,18 @@ def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): Returns: Returns true if the header is in alphabetical order. """ - # If previous section is different from current section, _last_header will - # be reset to empty string, so it's always less than current header. - # - # If previous line was a blank line, assume that the headers are - # intentionally sorted the way they are. - if (self._last_header > header_path and - Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): - return False - return True + # If previous section is different from current section, _last_header will + # be reset to empty string, so it's always less than current header. + # + # If previous line was a blank line, assume that the headers are + # intentionally sorted the way they are. + if (self._last_header > header_path and + Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): + return False + return True - def CheckNextIncludeOrder(self, header_type): - """Returns a non-empty error message if the next header is out of order. + def CheckNextIncludeOrder(self, header_type): + """Returns a non-empty error message if the next header is out of order. This function also updates the internal state to be ready to check the next include. @@ -730,80 +728,79 @@ def CheckNextIncludeOrder(self, header_type): error message describing what's wrong. """ - error_message = ('Found %s after %s' % - (self._TYPE_NAMES[header_type], - self._SECTION_NAMES[self._section])) - - last_section = self._section - - if header_type == _C_SYS_HEADER: - if self._section <= self._C_SECTION: - self._section = self._C_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _CPP_SYS_HEADER: - if self._section <= self._CPP_SECTION: - self._section = self._CPP_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _LIKELY_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - self._section = self._OTHER_H_SECTION - elif header_type == _POSSIBLE_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - # This will always be the fallback because we're not sure - # enough that the header is associated with this file. - self._section = self._OTHER_H_SECTION - else: - assert header_type == _OTHER_HEADER - self._section = self._OTHER_H_SECTION + error_message = ('Found %s after %s' % ( + self._TYPE_NAMES[header_type], self._SECTION_NAMES[self._section])) + + last_section = self._section + + if header_type == _C_SYS_HEADER: + if self._section <= self._C_SECTION: + self._section = self._C_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _CPP_SYS_HEADER: + if self._section <= self._CPP_SECTION: + self._section = self._CPP_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _LIKELY_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + self._section = self._OTHER_H_SECTION + elif header_type == _POSSIBLE_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + # This will always be the fallback because we're not sure + # enough that the header is associated with this file. + self._section = self._OTHER_H_SECTION + else: + assert header_type == _OTHER_HEADER + self._section = self._OTHER_H_SECTION - if last_section != self._section: - self._last_header = '' + if last_section != self._section: + self._last_header = '' - return '' + return '' class _CppLintState(object): - """Maintains module-wide state..""" - - def __init__(self): - self.verbose_level = 1 # global setting. - self.error_count = 0 # global count of reported errors - # filters to apply when emitting error messages - self.filters = _DEFAULT_FILTERS[:] - # backup of filter list. Used to restore the state after each file. - self._filters_backup = self.filters[:] - self.counting = 'total' # In what way are we counting errors? - self.errors_by_category = {} # string to int dict storing error counts - - # output format: - # "emacs" - format that emacs can parse (default) - # "vs7" - format that Microsoft Visual Studio 7 can parse - self.output_format = 'emacs' - - def SetOutputFormat(self, output_format): - """Sets the output format for errors.""" - self.output_format = output_format - - def SetVerboseLevel(self, level): - """Sets the module's verbosity, and returns the previous setting.""" - last_verbose_level = self.verbose_level - self.verbose_level = level - return last_verbose_level - - def SetCountingStyle(self, counting_style): - """Sets the module's counting options.""" - self.counting = counting_style - - def SetFilters(self, filters): - """Sets the error-message filters. + """Maintains module-wide state..""" + + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + # backup of filter list. Used to restore the state after each file. + self._filters_backup = self.filters[:] + self.counting = 'total' # In what way are we counting errors? + self.errors_by_category = {} # string to int dict storing error counts + + # output format: + # "emacs" - format that emacs can parse (default) + # "vs7" - format that Microsoft Visual Studio 7 can parse + self.output_format = 'emacs' + + def SetOutputFormat(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format + + def SetVerboseLevel(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level + + def SetCountingStyle(self, counting_style): + """Sets the module's counting options.""" + self.counting = counting_style + + def SetFilters(self, filters): + """Sets the error-message filters. These filters are applied when deciding whether to emit a given error message. @@ -816,86 +813,88 @@ def SetFilters(self, filters): ValueError: The comma-separated filters did not all start with '+' or '-'. E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" """ - # Default filters always have less priority than the flag ones. - self.filters = _DEFAULT_FILTERS[:] - self.AddFilters(filters) - - def AddFilters(self, filters): - """ Adds more filters to the existing list of error-message filters. """ - for filt in filters.split(','): - clean_filt = filt.strip() - if clean_filt: - self.filters.append(clean_filt) - for filt in self.filters: - if not (filt.startswith('+') or filt.startswith('-')): - raise ValueError('Every filter in --filters must start with + or -' - ' (%s does not)' % filt) - - def BackupFilters(self): - """ Saves the current filter list to backup storage.""" - self._filters_backup = self.filters[:] + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + self.AddFilters(filters) + + def AddFilters(self, filters): + """ Adds more filters to the existing list of error-message filters. """ + for filt in filters.split(','): + clean_filt = filt.strip() + if clean_filt: + self.filters.append(clean_filt) + for filt in self.filters: + if not (filt.startswith('+') or filt.startswith('-')): + raise ValueError( + 'Every filter in --filters must start with + or -' + ' (%s does not)' % filt) + + def BackupFilters(self): + """ Saves the current filter list to backup storage.""" + self._filters_backup = self.filters[:] + + def RestoreFilters(self): + """ Restores filters previously backed up.""" + self.filters = self._filters_backup[:] + + def ResetErrorCounts(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + self.errors_by_category = {} + + def IncrementErrorCount(self, category): + """Bumps the module's error statistic.""" + self.error_count += 1 + if self.counting in ('toplevel', 'detailed'): + if self.counting != 'detailed': + category = category.split('/')[0] + if category not in self.errors_by_category: + self.errors_by_category[category] = 0 + self.errors_by_category[category] += 1 + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" + for category, count in self.errors_by_category.iteritems(): + sys.stdout.write('Category \'%s\' errors found: %d\n' % + (category, count)) + sys.stdout.write('Total errors found: %d\n' % self.error_count) - def RestoreFilters(self): - """ Restores filters previously backed up.""" - self.filters = self._filters_backup[:] - - def ResetErrorCounts(self): - """Sets the module's error statistic back to zero.""" - self.error_count = 0 - self.errors_by_category = {} - - def IncrementErrorCount(self, category): - """Bumps the module's error statistic.""" - self.error_count += 1 - if self.counting in ('toplevel', 'detailed'): - if self.counting != 'detailed': - category = category.split('/')[0] - if category not in self.errors_by_category: - self.errors_by_category[category] = 0 - self.errors_by_category[category] += 1 - - def PrintErrorCounts(self): - """Print a summary of errors by category, and the total.""" - for category, count in self.errors_by_category.iteritems(): - sys.stdout.write('Category \'%s\' errors found: %d\n' % - (category, count)) - sys.stdout.write('Total errors found: %d\n' % self.error_count) _cpplint_state = _CppLintState() def _OutputFormat(): - """Gets the module's output format.""" - return _cpplint_state.output_format + """Gets the module's output format.""" + return _cpplint_state.output_format def _SetOutputFormat(output_format): - """Sets the module's output format.""" - _cpplint_state.SetOutputFormat(output_format) + """Sets the module's output format.""" + _cpplint_state.SetOutputFormat(output_format) def _VerboseLevel(): - """Returns the module's verbosity setting.""" - return _cpplint_state.verbose_level + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level def _SetVerboseLevel(level): - """Sets the module's verbosity, and returns the previous setting.""" - return _cpplint_state.SetVerboseLevel(level) + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.SetVerboseLevel(level) def _SetCountingStyle(level): - """Sets the module's counting options.""" - _cpplint_state.SetCountingStyle(level) + """Sets the module's counting options.""" + _cpplint_state.SetCountingStyle(level) def _Filters(): - """Returns the module's list of output filters, as a list.""" - return _cpplint_state.filters + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters def _SetFilters(filters): - """Sets the module's error-message filters. + """Sets the module's error-message filters. These filters are applied when deciding whether to emit a given error message. @@ -904,10 +903,11 @@ def _SetFilters(filters): filters: A string of comma-separated filters (eg "whitespace/indent"). Each filter should start with + or -; else we die. """ - _cpplint_state.SetFilters(filters) + _cpplint_state.SetFilters(filters) + def _AddFilters(filters): - """Adds more filter overrides. + """Adds more filter overrides. Unlike _SetFilters, this function does not reset the current list of filters available. @@ -916,93 +916,97 @@ def _AddFilters(filters): filters: A string of comma-separated filters (eg "whitespace/indent"). Each filter should start with + or -; else we die. """ - _cpplint_state.AddFilters(filters) + _cpplint_state.AddFilters(filters) + def _BackupFilters(): - """ Saves the current filter list to backup storage.""" - _cpplint_state.BackupFilters() + """ Saves the current filter list to backup storage.""" + _cpplint_state.BackupFilters() + def _RestoreFilters(): - """ Restores filters previously backed up.""" - _cpplint_state.RestoreFilters() + """ Restores filters previously backed up.""" + _cpplint_state.RestoreFilters() + class _FunctionState(object): - """Tracks current function name and the number of lines in its body.""" + """Tracks current function name and the number of lines in its body.""" - _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. - _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. - def __init__(self): - self.in_a_function = False - self.lines_in_function = 0 - self.current_function = '' + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' - def Begin(self, function_name): - """Start analyzing function body. + def Begin(self, function_name): + """Start analyzing function body. Args: function_name: The name of the function being tracked. """ - self.in_a_function = True - self.lines_in_function = 0 - self.current_function = function_name + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name - def Count(self): - """Count line in current function body.""" - if self.in_a_function: - self.lines_in_function += 1 + def Count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 - def Check(self, error, filename, linenum): - """Report if too many lines in function body. + def Check(self, error, filename, linenum): + """Report if too many lines in function body. Args: error: The function to call with any errors found. filename: The name of the current file. linenum: The number of the line to check. """ - if Match(r'T(EST|est)', self.current_function): - base_trigger = self._TEST_TRIGGER - else: - base_trigger = self._NORMAL_TRIGGER - trigger = base_trigger * 2**_VerboseLevel() - - if self.lines_in_function > trigger: - error_level = int(math.log(self.lines_in_function / base_trigger, 2)) - # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... - if error_level > 5: - error_level = 5 - error(filename, linenum, 'readability/fn_size', error_level, - 'Small and focused functions are preferred:' - ' %s has %d non-comment lines' - ' (error triggered by exceeding %d lines).' % ( - self.current_function, self.lines_in_function, trigger)) - - def End(self): - """Stop analyzing function body.""" - self.in_a_function = False + if Match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2**_VerboseLevel() + + if self.lines_in_function > trigger: + error_level = int( + math.log(self.lines_in_function / base_trigger, 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error(filename, linenum, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % ( + self.current_function, self.lines_in_function, trigger)) + + def End(self): + """Stop analyzing function body.""" + self.in_a_function = False class _IncludeError(Exception): - """Indicates a problem with the include order in a file.""" - pass + """Indicates a problem with the include order in a file.""" + pass class FileInfo(object): - """Provides utility functions for filenames. + """Provides utility functions for filenames. FileInfo provides easy access to the components of a file's path relative to the project root. """ - def __init__(self, filename): - self._filename = filename + def __init__(self, filename): + self._filename = filename - def FullName(self): - """Make Windows paths like Unix.""" - return os.path.abspath(self._filename).replace('\\', '/') + def FullName(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') - def RepositoryName(self): - """FullName after removing the local path to the repository. + def RepositoryName(self): + """FullName after removing the local path to the repository. If we have a real absolute path name here we can try to do something smart: detecting the root of the checkout and truncating /path/to/checkout from @@ -1011,43 +1015,43 @@ def RepositoryName(self): people on different computers who have checked the source out to different locations won't see bogus errors. """ - fullname = self.FullName() - - if os.path.exists(fullname): - project_dir = os.path.dirname(fullname) - - if os.path.exists(os.path.join(project_dir, ".svn")): - # If there's a .svn file in the current directory, we recursively look - # up the directory tree for the top of the SVN checkout - root_dir = project_dir - one_up_dir = os.path.dirname(root_dir) - while os.path.exists(os.path.join(one_up_dir, ".svn")): - root_dir = os.path.dirname(root_dir) - one_up_dir = os.path.dirname(one_up_dir) - - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by - # searching up from the current path. - root_dir = os.path.dirname(fullname) - while (root_dir != os.path.dirname(root_dir) and - not os.path.exists(os.path.join(root_dir, ".git")) and - not os.path.exists(os.path.join(root_dir, ".hg")) and - not os.path.exists(os.path.join(root_dir, ".svn"))): - root_dir = os.path.dirname(root_dir) - - if (os.path.exists(os.path.join(root_dir, ".git")) or - os.path.exists(os.path.join(root_dir, ".hg")) or - os.path.exists(os.path.join(root_dir, ".svn"))): - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Don't know what to do; header guard warnings may be wrong... - return fullname - - def Split(self): - """Splits the file into the directory, basename, and extension. + fullname = self.FullName() + + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) + + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we recursively look + # up the directory tree for the top of the SVN checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) + + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by + # searching up from the current path. + root_dir = os.path.dirname(fullname) + while (root_dir != os.path.dirname(root_dir) and + not os.path.exists(os.path.join(root_dir, ".git")) and + not os.path.exists(os.path.join(root_dir, ".hg")) and + not os.path.exists(os.path.join(root_dir, ".svn"))): + root_dir = os.path.dirname(root_dir) + + if (os.path.exists(os.path.join(root_dir, ".git")) or + os.path.exists(os.path.join(root_dir, ".hg")) or + os.path.exists(os.path.join(root_dir, ".svn"))): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Don't know what to do; header guard warnings may be wrong... + return fullname + + def Split(self): + """Splits the file into the directory, basename, and extension. For 'chrome/browser/browser.cc', Split() would return ('chrome/browser', 'browser', '.cc') @@ -1056,57 +1060,57 @@ def Split(self): A tuple of (directory, basename, extension). """ - googlename = self.RepositoryName() - project, rest = os.path.split(googlename) - return (project,) + os.path.splitext(rest) + googlename = self.RepositoryName() + project, rest = os.path.split(googlename) + return (project, ) + os.path.splitext(rest) - def BaseName(self): - """File base name - text after the final slash, before the final period.""" - return self.Split()[1] + def BaseName(self): + """File base name - text after the final slash, before the final period.""" + return self.Split()[1] - def Extension(self): - """File extension - text following the final period.""" - return self.Split()[2] + def Extension(self): + """File extension - text following the final period.""" + return self.Split()[2] - def NoExtension(self): - """File has no source file extension.""" - return '/'.join(self.Split()[0:2]) + def NoExtension(self): + """File has no source file extension.""" + return '/'.join(self.Split()[0:2]) - def IsSource(self): - """File has a source file extension.""" - return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') + def IsSource(self): + """File has a source file extension.""" + return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') def _ShouldPrintError(category, confidence, linenum): - """If confidence >= verbose, category passes filter and is not suppressed.""" + """If confidence >= verbose, category passes filter and is not suppressed.""" - # There are three ways we might decide not to print an error message: - # a "NOLINT(category)" comment appears in the source, - # the verbosity level isn't high enough, or the filters filter it out. - if IsErrorSuppressedByNolint(category, linenum): - return False + # There are three ways we might decide not to print an error message: + # a "NOLINT(category)" comment appears in the source, + # the verbosity level isn't high enough, or the filters filter it out. + if IsErrorSuppressedByNolint(category, linenum): + return False - if confidence < _cpplint_state.verbose_level: - return False + if confidence < _cpplint_state.verbose_level: + return False - is_filtered = False - for one_filter in _Filters(): - if one_filter.startswith('-'): - if category.startswith(one_filter[1:]): - is_filtered = True - elif one_filter.startswith('+'): - if category.startswith(one_filter[1:]): - is_filtered = False - else: - assert False # should have been checked for in SetFilter. - if is_filtered: - return False + is_filtered = False + for one_filter in _Filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in SetFilter. + if is_filtered: + return False - return True + return True def Error(filename, linenum, category, confidence, message): - """Logs the fact we've found a lint error. + """Logs the fact we've found a lint error. We log where the error was found, and also our confidence in the error, that is, how certain we are this is a legitimate style regression, and @@ -1127,17 +1131,17 @@ def Error(filename, linenum, category, confidence, message): and 1 meaning that it could be a legitimate construct. message: The error message. """ - if _ShouldPrintError(category, confidence, linenum): - _cpplint_state.IncrementErrorCount(category) - if _cpplint_state.output_format == 'vs7': - sys.stderr.write('%s(%s): %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - elif _cpplint_state.output_format == 'eclipse': - sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - else: - sys.stderr.write('%s:%s: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) + if _ShouldPrintError(category, confidence, linenum): + _cpplint_state.IncrementErrorCount(category) + if _cpplint_state.output_format == 'vs7': + sys.stderr.write('%s(%s): %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) + elif _cpplint_state.output_format == 'eclipse': + sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) + else: + sys.stderr.write('%s:%s: %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) # Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. @@ -1154,14 +1158,13 @@ def Error(filename, linenum, category, confidence, message): # if this doesn't work we try on left side but only if there's a non-character # on the right. _RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( - r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + - _RE_PATTERN_C_COMMENTS + r'\s+|' + - r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + + r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + _RE_PATTERN_C_COMMENTS + + r'\s+|' + r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + _RE_PATTERN_C_COMMENTS + r')') def IsCppString(line): - """Does line terminate so, that the next symbol is in string constant. + """Does line terminate so, that the next symbol is in string constant. This function does not consider single-line nor multi-line comments. @@ -1173,12 +1176,12 @@ def IsCppString(line): string constant. """ - line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" - return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 def CleanseRawStrings(raw_lines): - """Removes C++11 raw strings from lines. + """Removes C++11 raw strings from lines. Before: static const char kData[] = R"( @@ -1197,98 +1200,100 @@ def CleanseRawStrings(raw_lines): list of lines with C++11 raw strings replaced by empty strings. """ - delimiter = None - lines_without_raw_strings = [] - for line in raw_lines: - if delimiter: - # Inside a raw string, look for the end - end = line.find(delimiter) - if end >= 0: - # Found the end of the string, match leading space for this - # line and resume copying the original lines, and also insert - # a "" on the last line. - leading_space = Match(r'^(\s*)\S', line) - line = leading_space.group(1) + '""' + line[end + len(delimiter):] - delimiter = None - else: - # Haven't found the end yet, append a blank line. - line = '""' - - # Look for beginning of a raw string, and replace them with - # empty strings. This is done in a loop to handle multiple raw - # strings on the same line. - while delimiter is None: - # Look for beginning of a raw string. - # See 2.14.15 [lex.string] for syntax. - matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) - if matched: - delimiter = ')' + matched.group(2) + '"' - - end = matched.group(3).find(delimiter) - if end >= 0: - # Raw string ended on same line - line = (matched.group(1) + '""' + - matched.group(3)[end + len(delimiter):]) - delimiter = None - else: - # Start of a multi-line raw string - line = matched.group(1) + '""' - else: - break - - lines_without_raw_strings.append(line) - - # TODO(unknown): if delimiter is not None here, we might want to - # emit a warning for unterminated string. - return lines_without_raw_strings + delimiter = None + lines_without_raw_strings = [] + for line in raw_lines: + if delimiter: + # Inside a raw string, look for the end + end = line.find(delimiter) + if end >= 0: + # Found the end of the string, match leading space for this + # line and resume copying the original lines, and also insert + # a "" on the last line. + leading_space = Match(r'^(\s*)\S', line) + line = leading_space.group(1) + '""' + line[end + len( + delimiter):] + delimiter = None + else: + # Haven't found the end yet, append a blank line. + line = '""' + + # Look for beginning of a raw string, and replace them with + # empty strings. This is done in a loop to handle multiple raw + # strings on the same line. + while delimiter is None: + # Look for beginning of a raw string. + # See 2.14.15 [lex.string] for syntax. + matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', + line) + if matched: + delimiter = ')' + matched.group(2) + '"' + + end = matched.group(3).find(delimiter) + if end >= 0: + # Raw string ended on same line + line = (matched.group(1) + '""' + + matched.group(3)[end + len(delimiter):]) + delimiter = None + else: + # Start of a multi-line raw string + line = matched.group(1) + '""' + else: + break + + lines_without_raw_strings.append(line) + + # TODO(unknown): if delimiter is not None here, we might want to + # emit a warning for unterminated string. + return lines_without_raw_strings def FindNextMultiLineCommentStart(lines, lineix): - """Find the beginning marker for a multiline comment.""" - while lineix < len(lines): - if lines[lineix].strip().startswith('/*'): - # Only return this marker if the comment goes beyond this line - if lines[lineix].strip().find('*/', 2) < 0: - return lineix - lineix += 1 - return len(lines) + """Find the beginning marker for a multiline comment.""" + while lineix < len(lines): + if lines[lineix].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[lineix].strip().find('*/', 2) < 0: + return lineix + lineix += 1 + return len(lines) def FindNextMultiLineCommentEnd(lines, lineix): - """We are inside a comment, find the end marker.""" - while lineix < len(lines): - if lines[lineix].strip().endswith('*/'): - return lineix - lineix += 1 - return len(lines) + """We are inside a comment, find the end marker.""" + while lineix < len(lines): + if lines[lineix].strip().endswith('*/'): + return lineix + lineix += 1 + return len(lines) def RemoveMultiLineCommentsFromRange(lines, begin, end): - """Clears a range of lines for multi-line comments.""" - # Having // dummy comments makes the lines non-empty, so we will not get - # unnecessary blank line warnings later in the code. - for i in range(begin, end): - lines[i] = '/**/' + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '/**/' def RemoveMultiLineComments(filename, lines, error): - """Removes multiline (c-style) comments from lines.""" - lineix = 0 - while lineix < len(lines): - lineix_begin = FindNextMultiLineCommentStart(lines, lineix) - if lineix_begin >= len(lines): - return - lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) - if lineix_end >= len(lines): - error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, - 'Could not find end of multi-line comment') - return - RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) - lineix = lineix_end + 1 + """Removes multiline (c-style) comments from lines.""" + lineix = 0 + while lineix < len(lines): + lineix_begin = FindNextMultiLineCommentStart(lines, lineix) + if lineix_begin >= len(lines): + return + lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) + if lineix_end >= len(lines): + error(filename, lineix_begin + 1, 'readability/multiline_comment', + 5, 'Could not find end of multi-line comment') + return + RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) + lineix = lineix_end + 1 def CleanseComments(line): - """Removes //-comments and single-line C-style /* */ comments. + """Removes //-comments and single-line C-style /* */ comments. Args: line: A line of C++ source. @@ -1296,15 +1301,15 @@ def CleanseComments(line): Returns: The line with single-line comments removed. """ - commentpos = line.find('//') - if commentpos != -1 and not IsCppString(line[:commentpos]): - line = line[:commentpos].rstrip() - # get rid of /* ... */ - return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + commentpos = line.find('//') + if commentpos != -1 and not IsCppString(line[:commentpos]): + line = line[:commentpos].rstrip() + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) class CleansedLines(object): - """Holds 4 copies of all lines with different preprocessing applied to them. + """Holds 4 copies of all lines with different preprocessing applied to them. 1) elided member contains lines without strings and comments. 2) lines member contains lines without comments. @@ -1314,25 +1319,26 @@ class CleansedLines(object): All these members are of , and of the same length. """ - def __init__(self, lines): - self.elided = [] - self.lines = [] - self.raw_lines = lines - self.num_lines = len(lines) - self.lines_without_raw_strings = CleanseRawStrings(lines) - for linenum in range(len(self.lines_without_raw_strings)): - self.lines.append(CleanseComments( - self.lines_without_raw_strings[linenum])) - elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) - self.elided.append(CleanseComments(elided)) - - def NumLines(self): - """Returns the number of lines represented.""" - return self.num_lines - - @staticmethod - def _CollapseStrings(elided): - """Collapses strings and chars on a line to simple "" or '' blocks. + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self.num_lines = len(lines) + self.lines_without_raw_strings = CleanseRawStrings(lines) + for linenum in range(len(self.lines_without_raw_strings)): + self.lines.append( + CleanseComments(self.lines_without_raw_strings[linenum])) + elided = self._CollapseStrings(self.lines_without_raw_strings[ + linenum]) + self.elided.append(CleanseComments(elided)) + + def NumLines(self): + """Returns the number of lines represented.""" + return self.num_lines + + @staticmethod + def _CollapseStrings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. We nix strings first so we're not fooled by text like '"http://"' @@ -1342,64 +1348,65 @@ def _CollapseStrings(elided): Returns: The line with collapsed strings. """ - if _RE_PATTERN_INCLUDE.match(elided): - return elided - - # Remove escaped characters first to make quote/single quote collapsing - # basic. Things that look like escaped characters shouldn't occur - # outside of strings and chars. - elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) - - # Replace quoted strings and digit separators. Both single quotes - # and double quotes are processed in the same loop, otherwise - # nested quotes wouldn't work. - collapsed = '' - while True: - # Find the first quote character - match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) - if not match: - collapsed += elided - break - head, quote, tail = match.groups() - - if quote == '"': - # Collapse double quoted strings - second_quote = tail.find('"') - if second_quote >= 0: - collapsed += head + '""' - elided = tail[second_quote + 1:] - else: - # Unmatched double quote, don't bother processing the rest - # of the line since this is probably a multiline string. - collapsed += elided - break - else: - # Found single quote, check nearby text to eliminate digit separators. - # - # There is no special handling for floating point here, because - # the integer/fractional/exponent parts would all be parsed - # correctly as long as there are digits on both sides of the - # separator. So we are fine as long as we don't see something - # like "0.'3" (gcc 4.9.0 will not allow this literal). - if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): - match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail) - collapsed += head + match_literal.group(1).replace("'", '') - elided = match_literal.group(2) - else: - second_quote = tail.find('\'') - if second_quote >= 0: - collapsed += head + "''" - elided = tail[second_quote + 1:] - else: - # Unmatched single quote - collapsed += elided - break - - return collapsed + if _RE_PATTERN_INCLUDE.match(elided): + return elided + + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + + # Replace quoted strings and digit separators. Both single quotes + # and double quotes are processed in the same loop, otherwise + # nested quotes wouldn't work. + collapsed = '' + while True: + # Find the first quote character + match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) + if not match: + collapsed += elided + break + head, quote, tail = match.groups() + + if quote == '"': + # Collapse double quoted strings + second_quote = tail.find('"') + if second_quote >= 0: + collapsed += head + '""' + elided = tail[second_quote + 1:] + else: + # Unmatched double quote, don't bother processing the rest + # of the line since this is probably a multiline string. + collapsed += elided + break + else: + # Found single quote, check nearby text to eliminate digit separators. + # + # There is no special handling for floating point here, because + # the integer/fractional/exponent parts would all be parsed + # correctly as long as there are digits on both sides of the + # separator. So we are fine as long as we don't see something + # like "0.'3" (gcc 4.9.0 will not allow this literal). + if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): + match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', + "'" + tail) + collapsed += head + match_literal.group(1).replace("'", '') + elided = match_literal.group(2) + else: + second_quote = tail.find('\'') + if second_quote >= 0: + collapsed += head + "''" + elided = tail[second_quote + 1:] + else: + # Unmatched single quote + collapsed += elided + break + + return collapsed def FindEndOfExpressionInLine(line, startpos, stack): - """Find the position just after the end of current parenthesized expression. + """Find the position just after the end of current parenthesized expression. Args: line: a CleansedLines line. @@ -1411,73 +1418,73 @@ def FindEndOfExpressionInLine(line, startpos, stack): On finding an unclosed expression: (-1, None) Otherwise: (-1, new stack at end of this line) """ - for i in xrange(startpos, len(line)): - char = line[i] - if char in '([{': - # Found start of parenthesized expression, push to expression stack - stack.append(char) - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator - if stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - elif i > 0 and Search(r'\boperator\s*$', line[0:i]): - # operator<, don't add to stack - continue - else: - # Tentative start of template argument list - stack.append('<') - elif char in ')]}': - # Found end of parenthesized expression. - # - # If we are currently expecting a matching '>', the pending '<' - # must have been an operator. Remove them from expression stack. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - if ((stack[-1] == '(' and char == ')') or - (stack[-1] == '[' and char == ']') or - (stack[-1] == '{' and char == '}')): - stack.pop() - if not stack: - return (i + 1, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == '>': - # Found potential end of template argument list. - - # Ignore "->" and operator functions - if (i > 0 and - (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))): - continue - - # Pop the stack if there is a matching '<'. Otherwise, ignore - # this '>' since it must be an operator. - if stack: - if stack[-1] == '<': - stack.pop() - if not stack: - return (i + 1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '>', the matching '<' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - - # Did not find end of expression or unbalanced parentheses on this line - return (-1, stack) + for i in xrange(startpos, len(line)): + char = line[i] + if char in '([{': + # Found start of parenthesized expression, push to expression stack + stack.append(char) + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + if stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + elif i > 0 and Search(r'\boperator\s*$', line[0:i]): + # operator<, don't add to stack + continue + else: + # Tentative start of template argument list + stack.append('<') + elif char in ')]}': + # Found end of parenthesized expression. + # + # If we are currently expecting a matching '>', the pending '<' + # must have been an operator. Remove them from expression stack. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + if ((stack[-1] == '(' and char == ')') or + (stack[-1] == '[' and char == ']') or + (stack[-1] == '{' and char == '}')): + stack.pop() + if not stack: + return (i + 1, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == '>': + # Found potential end of template argument list. + + # Ignore "->" and operator functions + if (i > 0 and (line[i - 1] == '-' or Search(r'\boperator\s*$', + line[0:i - 1]))): + continue + + # Pop the stack if there is a matching '<'. Otherwise, ignore + # this '>' since it must be an operator. + if stack: + if stack[-1] == '<': + stack.pop() + if not stack: + return (i + 1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '>', the matching '<' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + + # Did not find end of expression or unbalanced parentheses on this line + return (-1, stack) def CloseExpression(clean_lines, linenum, pos): - """If input points to ( or { or [ or <, finds the position that closes it. + """If input points to ( or { or [ or <, finds the position that closes it. If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the linenum/pos that correspond to the closing of the expression. @@ -1499,29 +1506,29 @@ def CloseExpression(clean_lines, linenum, pos): 'cleansed' line at linenum. """ - line = clean_lines.elided[linenum] - if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): - return (line, clean_lines.NumLines(), -1) - - # Check first line - (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) - if end_pos > -1: - return (line, linenum, end_pos) - - # Continue scanning forward - while stack and linenum < clean_lines.NumLines() - 1: - linenum += 1 line = clean_lines.elided[linenum] - (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) + if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): + return (line, clean_lines.NumLines(), -1) + + # Check first line + (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) if end_pos > -1: - return (line, linenum, end_pos) + return (line, linenum, end_pos) + + # Continue scanning forward + while stack and linenum < clean_lines.NumLines() - 1: + linenum += 1 + line = clean_lines.elided[linenum] + (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) + if end_pos > -1: + return (line, linenum, end_pos) - # Did not find end of expression before end of file, give up - return (line, clean_lines.NumLines(), -1) + # Did not find end of expression before end of file, give up + return (line, clean_lines.NumLines(), -1) def FindStartOfExpressionInLine(line, endpos, stack): - """Find position at the matching start of current expression. + """Find position at the matching start of current expression. This is almost the reverse of FindEndOfExpressionInLine, but note that the input position and returned position differs by 1. @@ -1536,69 +1543,68 @@ def FindStartOfExpressionInLine(line, endpos, stack): On finding an unclosed expression: (-1, None) Otherwise: (-1, new stack at beginning of this line) """ - i = endpos - while i >= 0: - char = line[i] - if char in ')]}': - # Found end of expression, push to expression stack - stack.append(char) - elif char == '>': - # Found potential end of template argument list. - # - # Ignore it if it's a "->" or ">=" or "operator>" - if (i > 0 and - (line[i - 1] == '-' or - Match(r'\s>=\s', line[i - 1:]) or - Search(r'\boperator\s*$', line[0:i]))): - i -= 1 - else: - stack.append('>') - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator + i = endpos + while i >= 0: + char = line[i] + if char in ')]}': + # Found end of expression, push to expression stack + stack.append(char) + elif char == '>': + # Found potential end of template argument list. + # + # Ignore it if it's a "->" or ">=" or "operator>" + if (i > 0 and + (line[i - 1] == '-' or Match(r'\s>=\s', line[i - 1:]) or + Search(r'\boperator\s*$', line[0:i]))): + i -= 1 + else: + stack.append('>') + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + i -= 1 + else: + # If there is a matching '>', we can pop the expression stack. + # Otherwise, ignore this '<' since it must be an operator. + if stack and stack[-1] == '>': + stack.pop() + if not stack: + return (i, None) + elif char in '([{': + # Found start of expression. + # + # If there are any unmatched '>' on the stack, they must be + # operators. Remove those. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + if ((char == '(' and stack[-1] == ')') or + (char == '[' and stack[-1] == ']') or + (char == '{' and stack[-1] == '}')): + stack.pop() + if not stack: + return (i, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '<', the matching '>' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + i -= 1 - else: - # If there is a matching '>', we can pop the expression stack. - # Otherwise, ignore this '<' since it must be an operator. - if stack and stack[-1] == '>': - stack.pop() - if not stack: - return (i, None) - elif char in '([{': - # Found start of expression. - # - # If there are any unmatched '>' on the stack, they must be - # operators. Remove those. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) - if ((char == '(' and stack[-1] == ')') or - (char == '[' and stack[-1] == ']') or - (char == '{' and stack[-1] == '}')): - stack.pop() - if not stack: - return (i, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '<', the matching '>' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) - - i -= 1 - - return (-1, stack) + + return (-1, stack) def ReverseCloseExpression(clean_lines, linenum, pos): - """If input points to ) or } or ] or >, finds the position that opens it. + """If input points to ) or } or ] or >, finds the position that opens it. If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the linenum/pos that correspond to the opening of the expression. @@ -1614,42 +1620,42 @@ def ReverseCloseExpression(clean_lines, linenum, pos): we ignore strings and comments when matching; and the line we return is the 'cleansed' line at linenum. """ - line = clean_lines.elided[linenum] - if line[pos] not in ')}]>': - return (line, 0, -1) - - # Check last line - (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) - if start_pos > -1: - return (line, linenum, start_pos) - - # Continue scanning backward - while stack and linenum > 0: - linenum -= 1 line = clean_lines.elided[linenum] - (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack) - if start_pos > -1: - return (line, linenum, start_pos) + if line[pos] not in ')}]>': + return (line, 0, -1) - # Did not find start of expression before beginning of file, give up - return (line, 0, -1) + # Check last line + (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) + if start_pos > -1: + return (line, linenum, start_pos) + + # Continue scanning backward + while stack and linenum > 0: + linenum -= 1 + line = clean_lines.elided[linenum] + (start_pos, stack) = FindStartOfExpressionInLine(line, + len(line) - 1, stack) + if start_pos > -1: + return (line, linenum, start_pos) + + # Did not find start of expression before beginning of file, give up + return (line, 0, -1) def CheckForCopyright(filename, lines, error): - """Logs an error if no Copyright message appears at the top of the file.""" + """Logs an error if no Copyright message appears at the top of the file.""" - # We'll say it should occur by line 10. Don't forget there's a - # dummy line at the front. - for line in xrange(1, min(len(lines), 11)): - if re.search(r'Copyright', lines[line], re.I): break - else: # means no copyright line was found - error(filename, 0, 'legal/copyright', 5, - 'No copyright message found. ' - 'You should have a line: "Copyright [year] "') + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in xrange(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): break + else: # means no copyright line was found + error(filename, 0, 'legal/copyright', 5, 'No copyright message found. ' + 'You should have a line: "Copyright [year] "') def GetIndentLevel(line): - """Return the number of leading spaces in line. + """Return the number of leading spaces in line. Args: line: A string to check. @@ -1657,15 +1663,15 @@ def GetIndentLevel(line): Returns: An integer count of leading spaces, possibly zero. """ - indent = Match(r'^( *)\S', line) - if indent: - return len(indent.group(1)) - else: - return 0 + indent = Match(r'^( *)\S', line) + if indent: + return len(indent.group(1)) + else: + return 0 def GetHeaderGuardCPPVariable(filename): - """Returns the CPP variable that should be used as a header guard. + """Returns the CPP variable that should be used as a header guard. Args: filename: The name of a C++ header file. @@ -1675,12 +1681,12 @@ def GetHeaderGuardCPPVariable(filename): named file. """ - filename = os.path.basename(filename) - return re.sub(r'[^a-zA-Z0-9]', '_', filename).upper() + '_' + filename = os.path.basename(filename) + return re.sub(r'[^a-zA-Z0-9]', '_', filename).upper() + '_' def CheckForHeaderGuard(filename, clean_lines, error): - """Checks that the file contains a header guard. + """Checks that the file contains a header guard. Logs an error if no #ifndef header guard is present. For other headers, checks that the full pathname is used. @@ -1691,123 +1697,124 @@ def CheckForHeaderGuard(filename, clean_lines, error): error: The function to call with any errors found. """ - # Don't check for header guards if there are error suppression - # comments somewhere in this file. - # - # Because this is silencing a warning for a nonexistent line, we - # only support the very specific NOLINT(build/header_guard) syntax, - # and not the general NOLINT or NOLINT(*) syntax. - raw_lines = clean_lines.lines_without_raw_strings - for i in raw_lines: - if Search(r'//\s*NOLINT\(build/header_guard\)', i): - return - - cppvar = GetHeaderGuardCPPVariable(filename) - - ifndef = '' - ifndef_linenum = 0 - define = '' - endif = '' - endif_linenum = 0 - pragma_linenum = -1 - for linenum, line in enumerate(raw_lines): - linesplit = line.split() - if len(linesplit) >= 2: - if linesplit[0] == '#pragma' and linesplit[1] == 'once': - pragma_linenum = linenum - # find the first occurrence of #ifndef and #define, save arg - if not ifndef and linesplit[0] == '#ifndef': - # set ifndef to the header guard presented on the #ifndef line. - ifndef = linesplit[1] - ifndef_linenum = linenum - if not define and linesplit[0] == '#define': - define = linesplit[1] - # find the last occurrence of #endif, save entire line - if line.startswith('#endif'): - endif = line - endif_linenum = linenum - if pragma_linenum != -1: - return # short path for pragma once - if not ifndef or not define or ifndef != define: - error(filename, 0, 'build/header_guard', 5, - 'No #ifndef header guard found, suggested CPP variable is: %s' % - cppvar) - return - - # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ - # for backward compatibility. - if ifndef != cppvar: - error_level = 0 - if ifndef != cppvar + '_': - error_level = 5 - - ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum, + # Don't check for header guards if there are error suppression + # comments somewhere in this file. + # + # Because this is silencing a warning for a nonexistent line, we + # only support the very specific NOLINT(build/header_guard) syntax, + # and not the general NOLINT or NOLINT(*) syntax. + raw_lines = clean_lines.lines_without_raw_strings + for i in raw_lines: + if Search(r'//\s*NOLINT\(build/header_guard\)', i): + return + + cppvar = GetHeaderGuardCPPVariable(filename) + + ifndef = '' + ifndef_linenum = 0 + define = '' + endif = '' + endif_linenum = 0 + pragma_linenum = -1 + for linenum, line in enumerate(raw_lines): + linesplit = line.split() + if len(linesplit) >= 2: + if linesplit[0] == '#pragma' and linesplit[1] == 'once': + pragma_linenum = linenum + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and linesplit[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = linesplit[1] + ifndef_linenum = linenum + if not define and linesplit[0] == '#define': + define = linesplit[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_linenum = linenum + if pragma_linenum != -1: + return # short path for pragma once + if not ifndef or not define or ifndef != define: + error(filename, 0, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return + + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar: + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 + + ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], + ifndef_linenum, error) + error(filename, ifndef_linenum, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + # Check for "//" comments on endif line. + ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, error) - error(filename, ifndef_linenum, 'build/header_guard', error_level, - '#ifndef header guard has wrong style, please use: %s' % cppvar) - - # Check for "//" comments on endif line. - ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, - error) - match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) - if match: - if match.group(1) == '_': - # Issue low severity warning for deprecated double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif // %s"' % cppvar) - return - - # Didn't find the corresponding "//" comment. If this file does not - # contain any "//" comments at all, it could be that the compiler - # only wants "/**/" comments, look for those instead. - no_single_line_comments = True - for i in xrange(1, len(raw_lines) - 1): - line = raw_lines[i] - if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line): - no_single_line_comments = False - break - - if no_single_line_comments: - match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) if match: - if match.group(1) == '_': - # Low severity warning for double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif /* %s */"' % cppvar) - return + if match.group(1) == '_': + # Issue low severity warning for deprecated double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif // %s"' % cppvar) + return - # Didn't find anything - error(filename, endif_linenum, 'build/header_guard', 5, - '#endif line should be "#endif // %s"' % cppvar) + # Didn't find the corresponding "//" comment. If this file does not + # contain any "//" comments at all, it could be that the compiler + # only wants "/**/" comments, look for those instead. + no_single_line_comments = True + for i in xrange(1, len(raw_lines) - 1): + line = raw_lines[i] + if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', + line): + no_single_line_comments = False + break + + if no_single_line_comments: + match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + if match: + if match.group(1) == '_': + # Low severity warning for double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif /* %s */"' % cppvar) + return + + # Didn't find anything + error(filename, endif_linenum, 'build/header_guard', 5, + '#endif line should be "#endif // %s"' % cppvar) def CheckHeaderFileIncluded(filename, include_state, error): - """Logs an error if a .cc file does not include its header.""" - - # Do not check test files - if filename.endswith('_test.cc') or filename.endswith('_unittest.cc'): - return - - fileinfo = FileInfo(filename) - headerfile = filename[0:len(filename) - 2] + 'h' - if not os.path.exists(headerfile): - return - headername = FileInfo(headerfile).RepositoryName() - first_include = 0 - for section_list in include_state.include_list: - for f in section_list: - if headername in f[0] or f[0] in headername: + """Logs an error if a .cc file does not include its header.""" + + # Do not check test files + if filename.endswith('_test.cc') or filename.endswith('_unittest.cc'): + return + + fileinfo = FileInfo(filename) + headerfile = filename[0:len(filename) - 2] + 'h' + if not os.path.exists(headerfile): return - if not first_include: - first_include = f[1] + headername = FileInfo(headerfile).RepositoryName() + first_include = 0 + for section_list in include_state.include_list: + for f in section_list: + if headername in f[0] or f[0] in headername: + return + if not first_include: + first_include = f[1] - error(filename, first_include, 'build/include', 5, - '%s should include its header file %s' % (fileinfo.RepositoryName(), - headername)) + error(filename, first_include, 'build/include', 5, + '%s should include its header file %s' % (fileinfo.RepositoryName(), + headername)) def CheckForBadCharacters(filename, lines, error): - """Logs an error for each line containing bad characters. + """Logs an error for each line containing bad characters. Two kinds of bad characters: @@ -1823,16 +1830,19 @@ def CheckForBadCharacters(filename, lines, error): lines: An array of strings, each representing a line of the file. error: The function to call with any errors found. """ - for linenum, line in enumerate(lines): - if u'\ufffd' in line: - error(filename, linenum, 'readability/utf8', 5, - 'Line contains invalid UTF-8 (or Unicode replacement character).') - if '\0' in line: - error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') + for linenum, line in enumerate(lines): + if u'\ufffd' in line: + error( + filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).' + ) + if '\0' in line: + error(filename, linenum, 'readability/nul', 5, + 'Line contains NUL byte.') def CheckForNewlineAtEOF(filename, lines, error): - """Logs an error if there is no newline char at the end of the file. + """Logs an error if there is no newline char at the end of the file. Args: filename: The name of the current file. @@ -1840,17 +1850,18 @@ def CheckForNewlineAtEOF(filename, lines, error): error: The function to call with any errors found. """ - # The array lines() was created by adding two newlines to the - # original file (go figure), then splitting on \n. - # To verify that the file ends in \n, we just have to make sure the - # last-but-two element of lines() exists and is empty. - if len(lines) < 3 or lines[-2]: - error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, - 'Could not find a newline character at the end of the file.') + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, + len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): - """Logs an error if we see /* ... */ or "..." that extend past one line. + """Logs an error if we see /* ... */ or "..." that extend past one line. /* ... */ comments are legit inside macros, for one line. Otherwise, we prefer // comments, so it's ok to warn about the @@ -1866,25 +1877,25 @@ def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Remove all \\ (escaped backslashes) from the line. They are OK, and the - # second (escaped) slash may trigger later \" detection erroneously. - line = line.replace('\\\\', '') + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') - if line.count('/*') > line.count('*/'): - error(filename, linenum, 'readability/multiline_comment', 5, - 'Complex multi-line /*...*/-style comment found. ' - 'Lint may give bogus warnings. ' - 'Consider replacing these with //-style comments, ' - 'with #if 0...#endif, ' - 'or with more clearly structured multi-line comments.') + if line.count('/*') > line.count('*/'): + error(filename, linenum, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') - if (line.count('"') - line.count('\\"')) % 2: - error(filename, linenum, 'readability/multiline_string', 5, - 'Multi-line string ("...") found. This lint script doesn\'t ' - 'do well with such strings, and may give bogus warnings. ' - 'Use C++11 raw strings or concatenation instead.') + if (line.count('"') - line.count('\\"')) % 2: + error(filename, linenum, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. ' + 'Use C++11 raw strings or concatenation instead.') # (non-threadsafe name, thread-safe alternative, validation pattern) @@ -1911,14 +1922,12 @@ def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'), ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'), ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'), - ('strtok(', 'strtok_r(', - _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), - ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), - ) + ('strtok(', 'strtok_r(', _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), + ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), ) def CheckPosixThreading(filename, clean_lines, linenum, error): - """Checks for calls to thread-unsafe functions. + """Checks for calls to thread-unsafe functions. Much code has been originally written without consideration of multi-threading. Also, engineers are relying on their old experience; @@ -1932,19 +1941,18 @@ def CheckPosixThreading(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: - # Additional pattern matching check to confirm that this is the - # function we are looking for - if Search(pattern, line): - error(filename, linenum, 'runtime/threadsafe_fn', 2, - 'Consider using ' + multithread_safe_func + - '...) instead of ' + single_thread_func + - '...) for improved thread safety.') + line = clean_lines.elided[linenum] + for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: + # Additional pattern matching check to confirm that this is the + # function we are looking for + if Search(pattern, line): + error(filename, linenum, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_func + '...) instead of ' + + single_thread_func + '...) for improved thread safety.') def CheckVlogArguments(filename, clean_lines, linenum, error): - """Checks that VLOG() is only used for defining a logging level. + """Checks that VLOG() is only used for defining a logging level. For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and VLOG(FATAL) are not. @@ -1955,20 +1963,20 @@ def CheckVlogArguments(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): - error(filename, linenum, 'runtime/vlog', 5, - 'VLOG() should be used with numeric verbosity level. ' - 'Use LOG() if you want symbolic severity levels.') + line = clean_lines.elided[linenum] + if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): + error(filename, linenum, 'runtime/vlog', 5, + 'VLOG() should be used with numeric verbosity level. ' + 'Use LOG() if you want symbolic severity levels.') + # Matches invalid increment: *count++, which moves pointer instead of # incrementing a value. -_RE_PATTERN_INVALID_INCREMENT = re.compile( - r'^\s*\*\w+(\+\+|--);') +_RE_PATTERN_INVALID_INCREMENT = re.compile(r'^\s*\*\w+(\+\+|--);') def CheckInvalidIncrement(filename, clean_lines, linenum, error): - """Checks for invalid increment *count++. + """Checks for invalid increment *count++. For example following function: void increment_counter(int* count) { @@ -1983,37 +1991,38 @@ def CheckInvalidIncrement(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - if _RE_PATTERN_INVALID_INCREMENT.match(line): - error(filename, linenum, 'runtime/invalid_increment', 5, - 'Changing pointer instead of value (or unused value of operator*).') + line = clean_lines.elided[linenum] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error( + filename, linenum, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') def IsMacroDefinition(clean_lines, linenum): - if Search(r'^#define', clean_lines[linenum]): - return True + if Search(r'^#define', clean_lines[linenum]): + return True - if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): - return True + if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): + return True - return False + return False def IsForwardClassDeclaration(clean_lines, linenum): - return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) + return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) class _BlockInfo(object): - """Stores information about a generic block of code.""" + """Stores information about a generic block of code.""" - def __init__(self, seen_open_brace): - self.seen_open_brace = seen_open_brace - self.open_parentheses = 0 - self.inline_asm = _NO_ASM - self.check_namespace_indentation = False + def __init__(self, seen_open_brace): + self.seen_open_brace = seen_open_brace + self.open_parentheses = 0 + self.inline_asm = _NO_ASM + self.check_namespace_indentation = False - def CheckBegin(self, filename, clean_lines, linenum, error): - """Run checks that applies to text up to the opening brace. + def CheckBegin(self, filename, clean_lines, linenum, error): + """Run checks that applies to text up to the opening brace. This is mostly for checking the text after the class identifier and the "{", usually where the base class is specified. For other @@ -2025,10 +2034,10 @@ def CheckBegin(self, filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - pass + pass - def CheckEnd(self, filename, clean_lines, linenum, error): - """Run checks that applies to text after the closing brace. + def CheckEnd(self, filename, clean_lines, linenum, error): + """Run checks that applies to text after the closing brace. This is mostly used for checking end of namespace comments. @@ -2038,10 +2047,10 @@ def CheckEnd(self, filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - pass + pass - def IsBlockInfo(self): - """Returns true if this block is a _BlockInfo. + def IsBlockInfo(self): + """Returns true if this block is a _BlockInfo. This is convenient for verifying that an object is an instance of a _BlockInfo, but not an instance of any of the derived classes. @@ -2049,231 +2058,235 @@ def IsBlockInfo(self): Returns: True for this class, False for derived classes. """ - return self.__class__ == _BlockInfo + return self.__class__ == _BlockInfo class _ExternCInfo(_BlockInfo): - """Stores information about an 'extern "C"' block.""" + """Stores information about an 'extern "C"' block.""" - def __init__(self): - _BlockInfo.__init__(self, True) + def __init__(self): + _BlockInfo.__init__(self, True) class _ClassInfo(_BlockInfo): - """Stores information about a class.""" - - def __init__(self, name, class_or_struct, clean_lines, linenum): - _BlockInfo.__init__(self, False) - self.name = name - self.starting_linenum = linenum - self.is_derived = False - self.check_namespace_indentation = True - if class_or_struct == 'struct': - self.access = 'public' - self.is_struct = True - else: - self.access = 'private' - self.is_struct = False + """Stores information about a class.""" + + def __init__(self, name, class_or_struct, clean_lines, linenum): + _BlockInfo.__init__(self, False) + self.name = name + self.starting_linenum = linenum + self.is_derived = False + self.check_namespace_indentation = True + if class_or_struct == 'struct': + self.access = 'public' + self.is_struct = True + else: + self.access = 'private' + self.is_struct = False - # Remember initial indentation level for this class. Using raw_lines here - # instead of elided to account for leading comments. - self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) + # Remember initial indentation level for this class. Using raw_lines here + # instead of elided to account for leading comments. + self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) - # Try to find the end of the class. This will be confused by things like: - # class A { - # } *x = { ... - # - # But it's still good enough for CheckSectionSpacing. - self.last_line = 0 - depth = 0 - for i in range(linenum, clean_lines.NumLines()): - line = clean_lines.elided[i] - depth += line.count('{') - line.count('}') - if not depth: - self.last_line = i - break - - def CheckBegin(self, filename, clean_lines, linenum, error): - # Look for a bare ':' - if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): - self.is_derived = True - - def CheckEnd(self, filename, clean_lines, linenum, error): - # If there is a DISALLOW macro, it should appear near the end of - # the class. - seen_last_thing_in_class = False - for i in xrange(linenum - 1, self.starting_linenum, -1): - match = Search( - r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + - self.name + r'\)', - clean_lines.elided[i]) - if match: - if seen_last_thing_in_class: - error(filename, i, 'readability/constructors', 3, - match.group(1) + ' should be the last thing in the class') - break - - if not Match(r'^\s*$', clean_lines.elided[i]): - seen_last_thing_in_class = True - - # Check that closing brace is aligned with beginning of the class. - # Only do this if the closing brace is indented by only whitespaces. - # This means we will not check single-line class definitions. - indent = Match(r'^( *)\}', clean_lines.elided[linenum]) - if indent and len(indent.group(1)) != self.class_indent: - if self.is_struct: - parent = 'struct ' + self.name - else: - parent = 'class ' + self.name - error(filename, linenum, 'whitespace/indent', 3, - 'Closing brace should be aligned with beginning of %s' % parent) + # Try to find the end of the class. This will be confused by things like: + # class A { + # } *x = { ... + # + # But it's still good enough for CheckSectionSpacing. + self.last_line = 0 + depth = 0 + for i in range(linenum, clean_lines.NumLines()): + line = clean_lines.elided[i] + depth += line.count('{') - line.count('}') + if not depth: + self.last_line = i + break + + def CheckBegin(self, filename, clean_lines, linenum, error): + # Look for a bare ':' + if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): + self.is_derived = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + # If there is a DISALLOW macro, it should appear near the end of + # the class. + seen_last_thing_in_class = False + for i in xrange(linenum - 1, self.starting_linenum, -1): + match = Search( + r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + + self.name + r'\)', clean_lines.elided[i]) + if match: + if seen_last_thing_in_class: + error(filename, i, 'readability/constructors', 3, + match.group(1) + + ' should be the last thing in the class') + break + + if not Match(r'^\s*$', clean_lines.elided[i]): + seen_last_thing_in_class = True + + # Check that closing brace is aligned with beginning of the class. + # Only do this if the closing brace is indented by only whitespaces. + # This means we will not check single-line class definitions. + indent = Match(r'^( *)\}', clean_lines.elided[linenum]) + if indent and len(indent.group(1)) != self.class_indent: + if self.is_struct: + parent = 'struct ' + self.name + else: + parent = 'class ' + self.name + error(filename, linenum, 'whitespace/indent', 3, + 'Closing brace should be aligned with beginning of %s' % + parent) class _NamespaceInfo(_BlockInfo): - """Stores information about a namespace.""" - - def __init__(self, name, linenum): - _BlockInfo.__init__(self, False) - self.name = name or '' - self.starting_linenum = linenum - self.check_namespace_indentation = True - - def CheckEnd(self, filename, clean_lines, linenum, error): - """Check end of namespace comments.""" - line = clean_lines.raw_lines[linenum] - - # Check how many lines is enclosed in this namespace. Don't issue - # warning for missing namespace comments if there aren't enough - # lines. However, do apply checks if there is already an end of - # namespace comment and it's incorrect. - # - # TODO(unknown): We always want to check end of namespace comments - # if a namespace is large, but sometimes we also want to apply the - # check if a short namespace contained nontrivial things (something - # other than forward declarations). There is currently no logic on - # deciding what these nontrivial things are, so this check is - # triggered by namespace size only, which works most of the time. - if (linenum - self.starting_linenum < 10 - and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)): - return - - # Look for matching comment at end of namespace. - # - # Note that we accept C style "/* */" comments for terminating - # namespaces, so that code that terminate namespaces inside - # preprocessor macros can be cpplint clean. - # - # We also accept stuff like "// end of namespace ." with the - # period at the end. - # - # Besides these, we don't accept anything else, otherwise we might - # get false negatives when existing comment is a substring of the - # expected namespace. - if self.name: - # Named namespace - if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) + - r'[\*/\.\\\s]*$'), - line): - error(filename, linenum, 'readability/namespace', 5, - 'Namespace should be terminated with "// namespace %s"' % - self.name) - else: - # Anonymous namespace - if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): - # If "// namespace anonymous" or "// anonymous namespace (more text)", - # mention "// anonymous namespace" as an acceptable form - if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', line): - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"' - ' or "// anonymous namespace"') + """Stores information about a namespace.""" + + def __init__(self, name, linenum): + _BlockInfo.__init__(self, False) + self.name = name or '' + self.starting_linenum = linenum + self.check_namespace_indentation = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Check end of namespace comments.""" + line = clean_lines.raw_lines[linenum] + + # Check how many lines is enclosed in this namespace. Don't issue + # warning for missing namespace comments if there aren't enough + # lines. However, do apply checks if there is already an end of + # namespace comment and it's incorrect. + # + # TODO(unknown): We always want to check end of namespace comments + # if a namespace is large, but sometimes we also want to apply the + # check if a short namespace contained nontrivial things (something + # other than forward declarations). There is currently no logic on + # deciding what these nontrivial things are, so this check is + # triggered by namespace size only, which works most of the time. + if (linenum - self.starting_linenum < 10 and + not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)): + return + + # Look for matching comment at end of namespace. + # + # Note that we accept C style "/* */" comments for terminating + # namespaces, so that code that terminate namespaces inside + # preprocessor macros can be cpplint clean. + # + # We also accept stuff like "// end of namespace ." with the + # period at the end. + # + # Besides these, we don't accept anything else, otherwise we might + # get false negatives when existing comment is a substring of the + # expected namespace. + if self.name: + # Named namespace + if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + + re.escape(self.name) + r'[\*/\.\\\s]*$'), line): + error(filename, linenum, 'readability/namespace', 5, + 'Namespace should be terminated with "// namespace %s"' % + self.name) else: - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"') + # Anonymous namespace + if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): + # If "// namespace anonymous" or "// anonymous namespace (more text)", + # mention "// anonymous namespace" as an acceptable form + if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', + line): + error( + filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ' or "// anonymous namespace"') + else: + error( + filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ) class _PreprocessorInfo(object): - """Stores checkpoints of nesting stacks when #if/#else is seen.""" + """Stores checkpoints of nesting stacks when #if/#else is seen.""" - def __init__(self, stack_before_if): - # The entire nesting stack before #if - self.stack_before_if = stack_before_if + def __init__(self, stack_before_if): + # The entire nesting stack before #if + self.stack_before_if = stack_before_if - # The entire nesting stack up to #else - self.stack_before_else = [] + # The entire nesting stack up to #else + self.stack_before_else = [] - # Whether we have already seen #else or #elif - self.seen_else = False + # Whether we have already seen #else or #elif + self.seen_else = False class NestingState(object): - """Holds states related to parsing braces.""" - - def __init__(self): - # Stack for tracking all braces. An object is pushed whenever we - # see a "{", and popped when we see a "}". Only 3 types of - # objects are possible: - # - _ClassInfo: a class or struct. - # - _NamespaceInfo: a namespace. - # - _BlockInfo: some other type of block. - self.stack = [] - - # Top of the previous stack before each Update(). - # - # Because the nesting_stack is updated at the end of each line, we - # had to do some convoluted checks to find out what is the current - # scope at the beginning of the line. This check is simplified by - # saving the previous top of nesting stack. - # - # We could save the full stack, but we only need the top. Copying - # the full nesting stack would slow down cpplint by ~10%. - self.previous_stack_top = [] + """Holds states related to parsing braces.""" + + def __init__(self): + # Stack for tracking all braces. An object is pushed whenever we + # see a "{", and popped when we see a "}". Only 3 types of + # objects are possible: + # - _ClassInfo: a class or struct. + # - _NamespaceInfo: a namespace. + # - _BlockInfo: some other type of block. + self.stack = [] + + # Top of the previous stack before each Update(). + # + # Because the nesting_stack is updated at the end of each line, we + # had to do some convoluted checks to find out what is the current + # scope at the beginning of the line. This check is simplified by + # saving the previous top of nesting stack. + # + # We could save the full stack, but we only need the top. Copying + # the full nesting stack would slow down cpplint by ~10%. + self.previous_stack_top = [] - # Stack of _PreprocessorInfo objects. - self.pp_stack = [] + # Stack of _PreprocessorInfo objects. + self.pp_stack = [] - def SeenOpenBrace(self): - """Check if we have seen the opening brace for the innermost block. + def SeenOpenBrace(self): + """Check if we have seen the opening brace for the innermost block. Returns: True if we have seen the opening brace, False if the innermost block is still expecting an opening brace. """ - return (not self.stack) or self.stack[-1].seen_open_brace + return (not self.stack) or self.stack[-1].seen_open_brace - def InNamespaceBody(self): - """Check if we are currently one level inside a namespace body. + def InNamespaceBody(self): + """Check if we are currently one level inside a namespace body. Returns: True if top of the stack is a namespace block, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _NamespaceInfo) + return self.stack and isinstance(self.stack[-1], _NamespaceInfo) - def InExternC(self): - """Check if we are currently one level inside an 'extern "C"' block. + def InExternC(self): + """Check if we are currently one level inside an 'extern "C"' block. Returns: True if top of the stack is an extern block, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _ExternCInfo) + return self.stack and isinstance(self.stack[-1], _ExternCInfo) - def InClassDeclaration(self): - """Check if we are currently one level inside a class or struct declaration. + def InClassDeclaration(self): + """Check if we are currently one level inside a class or struct declaration. Returns: True if top of the stack is a class/struct, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _ClassInfo) + return self.stack and isinstance(self.stack[-1], _ClassInfo) - def InAsmBlock(self): - """Check if we are currently one level inside an inline ASM block. + def InAsmBlock(self): + """Check if we are currently one level inside an inline ASM block. Returns: True if the top of the stack is a block containing inline ASM. """ - return self.stack and self.stack[-1].inline_asm != _NO_ASM + return self.stack and self.stack[-1].inline_asm != _NO_ASM - def InTemplateArgumentList(self, clean_lines, linenum, pos): - """Check if current position is inside template argument list. + def InTemplateArgumentList(self, clean_lines, linenum, pos): + """Check if current position is inside template argument list. Args: clean_lines: A CleansedLines instance containing the file. @@ -2282,50 +2295,51 @@ def InTemplateArgumentList(self, clean_lines, linenum, pos): Returns: True if (linenum, pos) is inside template arguments. """ - while linenum < clean_lines.NumLines(): - # Find the earliest character that might indicate a template argument - line = clean_lines.elided[linenum] - match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) - if not match: - linenum += 1 - pos = 0 - continue - token = match.group(1) - pos += len(match.group(0)) - - # These things do not look like template argument list: - # class Suspect { - # class Suspect x; } - if token in ('{', '}', ';'): return False - - # These things look like template argument list: - # template - # template - # template - # template - if token in ('>', '=', '[', ']', '.'): return True - - # Check if token is an unmatched '<'. - # If not, move on to the next character. - if token != '<': - pos += 1 - if pos >= len(line): - linenum += 1 - pos = 0 - continue - - # We can't be sure if we just find a single '<', and need to - # find the matching '>'. - (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1) - if end_pos < 0: - # Not sure if template argument list or syntax error in file + while linenum < clean_lines.NumLines(): + # Find the earliest character that might indicate a template argument + line = clean_lines.elided[linenum] + match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) + if not match: + linenum += 1 + pos = 0 + continue + token = match.group(1) + pos += len(match.group(0)) + + # These things do not look like template argument list: + # class Suspect { + # class Suspect x; } + if token in ('{', '}', ';'): return False + + # These things look like template argument list: + # template + # template + # template + # template + if token in ('>', '=', '[', ']', '.'): return True + + # Check if token is an unmatched '<'. + # If not, move on to the next character. + if token != '<': + pos += 1 + if pos >= len(line): + linenum += 1 + pos = 0 + continue + + # We can't be sure if we just find a single '<', and need to + # find the matching '>'. + (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, + pos - 1) + if end_pos < 0: + # Not sure if template argument list or syntax error in file + return False + linenum = end_line + pos = end_pos return False - linenum = end_line - pos = end_pos - return False - def UpdatePreprocessor(self, line): - """Update preprocessor stack. + def UpdatePreprocessor(self, line): + """Update preprocessor stack. We need to handle preprocessors due to classes like this: #ifdef SWIG @@ -2345,44 +2359,45 @@ def UpdatePreprocessor(self, line): Args: line: current line to check. """ - if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): - # Beginning of #if block, save the nesting stack here. The saved - # stack will allow us to restore the parsing state in the #else case. - self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) - elif Match(r'^\s*#\s*(else|elif)\b', line): - # Beginning of #else block - if self.pp_stack: - if not self.pp_stack[-1].seen_else: - # This is the first #else or #elif block. Remember the - # whole nesting stack up to this point. This is what we - # keep after the #endif. - self.pp_stack[-1].seen_else = True - self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) - - # Restore the stack to how it was before the #if - self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) - else: - # TODO(unknown): unexpected #else, issue warning? - pass - elif Match(r'^\s*#\s*endif\b', line): - # End of #if or #else blocks. - if self.pp_stack: - # If we saw an #else, we will need to restore the nesting - # stack to its former state before the #else, otherwise we - # will just continue from where we left off. - if self.pp_stack[-1].seen_else: - # Here we can just use a shallow copy since we are the last - # reference to it. - self.stack = self.pp_stack[-1].stack_before_else - # Drop the corresponding #if - self.pp_stack.pop() - else: - # TODO(unknown): unexpected #endif, issue warning? - pass - - # TODO(unknown): Update() is too long, but we will refactor later. - def Update(self, filename, clean_lines, linenum, error): - """Update nesting state with current line. + if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): + # Beginning of #if block, save the nesting stack here. The saved + # stack will allow us to restore the parsing state in the #else case. + self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) + elif Match(r'^\s*#\s*(else|elif)\b', line): + # Beginning of #else block + if self.pp_stack: + if not self.pp_stack[-1].seen_else: + # This is the first #else or #elif block. Remember the + # whole nesting stack up to this point. This is what we + # keep after the #endif. + self.pp_stack[-1].seen_else = True + self.pp_stack[-1].stack_before_else = copy.deepcopy( + self.stack) + + # Restore the stack to how it was before the #if + self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) + else: + # TODO(unknown): unexpected #else, issue warning? + pass + elif Match(r'^\s*#\s*endif\b', line): + # End of #if or #else blocks. + if self.pp_stack: + # If we saw an #else, we will need to restore the nesting + # stack to its former state before the #else, otherwise we + # will just continue from where we left off. + if self.pp_stack[-1].seen_else: + # Here we can just use a shallow copy since we are the last + # reference to it. + self.stack = self.pp_stack[-1].stack_before_else + # Drop the corresponding #if + self.pp_stack.pop() + else: + # TODO(unknown): unexpected #endif, issue warning? + pass + + # TODO(unknown): Update() is too long, but we will refactor later. + def Update(self, filename, clean_lines, linenum, error): + """Update nesting state with current line. Args: filename: The name of the current file. @@ -2390,198 +2405,201 @@ def Update(self, filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Remember top of the previous nesting stack. - # - # The stack is always pushed/popped and not modified in place, so - # we can just do a shallow copy instead of copy.deepcopy. Using - # deepcopy would slow down cpplint by ~28%. - if self.stack: - self.previous_stack_top = self.stack[-1] - else: - self.previous_stack_top = None - - # Update pp_stack - self.UpdatePreprocessor(line) - - # Count parentheses. This is to avoid adding struct arguments to - # the nesting stack. - if self.stack: - inner_block = self.stack[-1] - depth_change = line.count('(') - line.count(')') - inner_block.open_parentheses += depth_change - - # Also check if we are starting or ending an inline assembly block. - if inner_block.inline_asm in (_NO_ASM, _END_ASM): - if (depth_change != 0 and - inner_block.open_parentheses == 1 and - _MATCH_ASM.match(line)): - # Enter assembly block - inner_block.inline_asm = _INSIDE_ASM - else: - # Not entering assembly block. If previous line was _END_ASM, - # we will now shift to _NO_ASM state. - inner_block.inline_asm = _NO_ASM - elif (inner_block.inline_asm == _INSIDE_ASM and - inner_block.open_parentheses == 0): - # Exit assembly block - inner_block.inline_asm = _END_ASM - - # Consume namespace declaration at the beginning of the line. Do - # this in a loop so that we catch same line declarations like this: - # namespace proto2 { namespace bridge { class MessageSet; } } - while True: - # Match start of namespace. The "\b\s*" below catches namespace - # declarations even if it weren't followed by a whitespace, this - # is so that we don't confuse our namespace checker. The - # missing spaces will be flagged by CheckSpacing. - namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) - if not namespace_decl_match: - break - - new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) - self.stack.append(new_namespace) - - line = namespace_decl_match.group(2) - if line.find('{') != -1: - new_namespace.seen_open_brace = True - line = line[line.find('{') + 1:] - - # Look for a class declaration in whatever is left of the line - # after parsing namespaces. The regexp accounts for decorated classes - # such as in: - # class LOCKABLE API Object { - # }; - class_decl_match = Match( - r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?' - r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' - r'(.*)$', line) - if (class_decl_match and - (not self.stack or self.stack[-1].open_parentheses == 0)): - # We do not want to accept classes that are actually template arguments: - # template , - # template class Ignore3> - # void Function() {}; - # - # To avoid template argument cases, we scan forward and look for - # an unmatched '>'. If we see one, assume we are inside a - # template argument list. - end_declaration = len(class_decl_match.group(1)) - if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration): - self.stack.append(_ClassInfo( - class_decl_match.group(3), class_decl_match.group(2), - clean_lines, linenum)) - line = class_decl_match.group(4) - - # If we have not yet seen the opening brace for the innermost block, - # run checks here. - if not self.SeenOpenBrace(): - self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) - - # Update access control if we are inside a class/struct - if self.stack and isinstance(self.stack[-1], _ClassInfo): - classinfo = self.stack[-1] - access_match = Match( - r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' - r':(?:[^:]|$)', - line) - if access_match: - classinfo.access = access_match.group(2) - - # Check that access keywords are indented +1 space. Skip this - # check if the keywords are not preceded by whitespaces. - indent = access_match.group(1) - if (len(indent) != classinfo.class_indent + 1 and - Match(r'^\s*$', indent)): - if classinfo.is_struct: - parent = 'struct ' + classinfo.name - else: - parent = 'class ' + classinfo.name - slots = '' - if access_match.group(3): - slots = access_match.group(3) - error(filename, linenum, 'whitespace/indent', 3, - '%s%s: should be indented +1 space inside %s' % ( - access_match.group(2), slots, parent)) - - # Consume braces or semicolons from what's left of the line - while True: - # Match first brace, semicolon, or closed parenthesis. - matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) - if not matched: - break - - token = matched.group(1) - if token == '{': - # If namespace or class hasn't seen a opening brace yet, mark - # namespace/class head as complete. Push a new block onto the - # stack otherwise. - if not self.SeenOpenBrace(): - self.stack[-1].seen_open_brace = True - elif Match(r'^extern\s*"[^"]*"\s*\{', line): - self.stack.append(_ExternCInfo()) - else: - self.stack.append(_BlockInfo(True)) - if _MATCH_ASM.match(line): - self.stack[-1].inline_asm = _BLOCK_ASM - - elif token == ';' or token == ')': - # If we haven't seen an opening brace yet, but we already saw - # a semicolon, this is probably a forward declaration. Pop - # the stack for these. + # Remember top of the previous nesting stack. # - # Similarly, if we haven't seen an opening brace yet, but we - # already saw a closing parenthesis, then these are probably - # function arguments with extra "class" or "struct" keywords. - # Also pop these stack for these. - if not self.SeenOpenBrace(): - self.stack.pop() - else: # token == '}' - # Perform end of block checks and pop the stack. + # The stack is always pushed/popped and not modified in place, so + # we can just do a shallow copy instead of copy.deepcopy. Using + # deepcopy would slow down cpplint by ~28%. if self.stack: - self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) - self.stack.pop() - line = matched.group(2) + self.previous_stack_top = self.stack[-1] + else: + self.previous_stack_top = None + + # Update pp_stack + self.UpdatePreprocessor(line) - def InnermostClass(self): - """Get class info on the top of the stack. + # Count parentheses. This is to avoid adding struct arguments to + # the nesting stack. + if self.stack: + inner_block = self.stack[-1] + depth_change = line.count('(') - line.count(')') + inner_block.open_parentheses += depth_change + + # Also check if we are starting or ending an inline assembly block. + if inner_block.inline_asm in (_NO_ASM, _END_ASM): + if (depth_change != 0 and inner_block.open_parentheses == 1 and + _MATCH_ASM.match(line)): + # Enter assembly block + inner_block.inline_asm = _INSIDE_ASM + else: + # Not entering assembly block. If previous line was _END_ASM, + # we will now shift to _NO_ASM state. + inner_block.inline_asm = _NO_ASM + elif (inner_block.inline_asm == _INSIDE_ASM and + inner_block.open_parentheses == 0): + # Exit assembly block + inner_block.inline_asm = _END_ASM + + # Consume namespace declaration at the beginning of the line. Do + # this in a loop so that we catch same line declarations like this: + # namespace proto2 { namespace bridge { class MessageSet; } } + while True: + # Match start of namespace. The "\b\s*" below catches namespace + # declarations even if it weren't followed by a whitespace, this + # is so that we don't confuse our namespace checker. The + # missing spaces will be flagged by CheckSpacing. + namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', + line) + if not namespace_decl_match: + break + + new_namespace = _NamespaceInfo( + namespace_decl_match.group(1), linenum) + self.stack.append(new_namespace) + + line = namespace_decl_match.group(2) + if line.find('{') != -1: + new_namespace.seen_open_brace = True + line = line[line.find('{') + 1:] + + # Look for a class declaration in whatever is left of the line + # after parsing namespaces. The regexp accounts for decorated classes + # such as in: + # class LOCKABLE API Object { + # }; + class_decl_match = Match( + r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?' + r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' + r'(.*)$', line) + if (class_decl_match and + (not self.stack or self.stack[-1].open_parentheses == 0)): + # We do not want to accept classes that are actually template arguments: + # template , + # template class Ignore3> + # void Function() {}; + # + # To avoid template argument cases, we scan forward and look for + # an unmatched '>'. If we see one, assume we are inside a + # template argument list. + end_declaration = len(class_decl_match.group(1)) + if not self.InTemplateArgumentList(clean_lines, linenum, + end_declaration): + self.stack.append( + _ClassInfo( + class_decl_match.group(3), + class_decl_match.group(2), clean_lines, linenum)) + line = class_decl_match.group(4) + + # If we have not yet seen the opening brace for the innermost block, + # run checks here. + if not self.SeenOpenBrace(): + self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) + + # Update access control if we are inside a class/struct + if self.stack and isinstance(self.stack[-1], _ClassInfo): + classinfo = self.stack[-1] + access_match = Match( + r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' + r':(?:[^:]|$)', line) + if access_match: + classinfo.access = access_match.group(2) + + # Check that access keywords are indented +1 space. Skip this + # check if the keywords are not preceded by whitespaces. + indent = access_match.group(1) + if (len(indent) != classinfo.class_indent + 1 and + Match(r'^\s*$', indent)): + if classinfo.is_struct: + parent = 'struct ' + classinfo.name + else: + parent = 'class ' + classinfo.name + slots = '' + if access_match.group(3): + slots = access_match.group(3) + error(filename, linenum, 'whitespace/indent', 3, + '%s%s: should be indented +1 space inside %s' % ( + access_match.group(2), slots, parent)) + + # Consume braces or semicolons from what's left of the line + while True: + # Match first brace, semicolon, or closed parenthesis. + matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) + if not matched: + break + + token = matched.group(1) + if token == '{': + # If namespace or class hasn't seen a opening brace yet, mark + # namespace/class head as complete. Push a new block onto the + # stack otherwise. + if not self.SeenOpenBrace(): + self.stack[-1].seen_open_brace = True + elif Match(r'^extern\s*"[^"]*"\s*\{', line): + self.stack.append(_ExternCInfo()) + else: + self.stack.append(_BlockInfo(True)) + if _MATCH_ASM.match(line): + self.stack[-1].inline_asm = _BLOCK_ASM + + elif token == ';' or token == ')': + # If we haven't seen an opening brace yet, but we already saw + # a semicolon, this is probably a forward declaration. Pop + # the stack for these. + # + # Similarly, if we haven't seen an opening brace yet, but we + # already saw a closing parenthesis, then these are probably + # function arguments with extra "class" or "struct" keywords. + # Also pop these stack for these. + if not self.SeenOpenBrace(): + self.stack.pop() + else: # token == '}' + # Perform end of block checks and pop the stack. + if self.stack: + self.stack[-1].CheckEnd(filename, clean_lines, linenum, + error) + self.stack.pop() + line = matched.group(2) + + def InnermostClass(self): + """Get class info on the top of the stack. Returns: A _ClassInfo object if we are inside a class, or None otherwise. """ - for i in range(len(self.stack), 0, -1): - classinfo = self.stack[i - 1] - if isinstance(classinfo, _ClassInfo): - return classinfo - return None + for i in range(len(self.stack), 0, -1): + classinfo = self.stack[i - 1] + if isinstance(classinfo, _ClassInfo): + return classinfo + return None - def CheckCompletedBlocks(self, filename, error): - """Checks that all classes and namespaces have been completely parsed. + def CheckCompletedBlocks(self, filename, error): + """Checks that all classes and namespaces have been completely parsed. Call this when all lines in a file have been processed. Args: filename: The name of the current file. error: The function to call with any errors found. """ - # Note: This test can result in false positives if #ifdef constructs - # get in the way of brace matching. See the testBuildClass test in - # cpplint_unittest.py for an example of this. - for obj in self.stack: - if isinstance(obj, _ClassInfo): - error(filename, obj.starting_linenum, 'build/class', 5, - 'Failed to find complete declaration of class %s' % - obj.name) - elif isinstance(obj, _NamespaceInfo): - error(filename, obj.starting_linenum, 'build/namespaces', 5, - 'Failed to find complete declaration of namespace %s' % - obj.name) - - -def CheckForNonStandardConstructs(filename, clean_lines, linenum, - nesting_state, error): - r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + for obj in self.stack: + if isinstance(obj, _ClassInfo): + error(filename, obj.starting_linenum, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + obj.name) + elif isinstance(obj, _NamespaceInfo): + error(filename, obj.starting_linenum, 'build/namespaces', 5, + 'Failed to find complete declaration of namespace %s' % + obj.name) + + +def CheckForNonStandardConstructs(filename, clean_lines, linenum, nesting_state, + error): + r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. Complain about several constructs which gcc-2 accepts, but which are not standard C++. Warning about these in lint is one way to ease the @@ -2608,143 +2626,144 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum, filename, line number, error level, and message """ - # Remove comments from the line, but leave in strings for now. - line = clean_lines.lines[linenum] - - if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): - error(filename, linenum, 'runtime/printf_format', 3, - '%q in format strings is deprecated. Use %ll instead.') - - if Search(r'printf\s*\(.*".*%\d+\$', line): - error(filename, linenum, 'runtime/printf_format', 2, - '%N$ formats are unconventional. Try rewriting to avoid them.') - - # Remove escaped backslashes before looking for undefined escapes. - line = line.replace('\\\\', '') - - if Search(r'("|\').*\\(%|\[|\(|{)', line): - error(filename, linenum, 'build/printf_format', 3, - '%, [, (, and { are undefined character escapes. Unescape them.') - - # For the rest, work with both comments and strings removed. - line = clean_lines.elided[linenum] - - if Search(r'\b(const|volatile|void|char|short|int|long' - r'|float|double|signed|unsigned' - r'|schar|u?int8|u?int16|u?int32|u?int64)' - r'\s+(register|static|extern|typedef)\b', - line): - error(filename, linenum, 'build/storage_class', 5, - 'Storage class (static, extern, typedef, etc) should be first.') - - if Match(r'\s*#\s*endif\s*[^/\s]+', line): - error(filename, linenum, 'build/endif_comment', 5, - 'Uncommented text after #endif is non-standard. Use a comment.') - - if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): - error(filename, linenum, 'build/forward_decl', 5, - 'Inner-style forward declarations are invalid. Remove this line.') - - if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', - line): - error(filename, linenum, 'build/deprecated', 3, - '>? and ))?' - # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' - error(filename, linenum, 'runtime/member_string_references', 2, - 'const string& members are dangerous. It is much better to use ' - 'alternatives, such as pointers or simple constants.') - - # Everything else in this function operates on class declarations. - # Return early if the top of the nesting stack is not a class, or if - # the class head is not completed yet. - classinfo = nesting_state.InnermostClass() - if not classinfo or not classinfo.seen_open_brace: - return - - # The class may have been declared with namespace or classname qualifiers. - # The constructor and destructor will not have those qualifiers. - base_classname = classinfo.name.split('::')[-1] - - # Look for single-argument constructors that aren't marked explicit. - # Technically a valid construct, but against style. Also look for - # non-single-argument constructors which are also technically valid, but - # strongly suggest something is wrong. - explicit_constructor_match = Match( - r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*' - r'\(((?:[^()]|\([^()]*\))*)\)' - % re.escape(base_classname), - line) - - if explicit_constructor_match: - is_marked_explicit = explicit_constructor_match.group(1) - - if not explicit_constructor_match.group(2): - constructor_args = [] - else: - constructor_args = explicit_constructor_match.group(2).split(',') - - # collapse arguments so that commas in template parameter lists and function - # argument parameter lists don't split arguments in two - i = 0 - while i < len(constructor_args): - constructor_arg = constructor_args[i] - while (constructor_arg.count('<') > constructor_arg.count('>') or - constructor_arg.count('(') > constructor_arg.count(')')): - constructor_arg += ',' + constructor_args[i + 1] - del constructor_args[i + 1] - constructor_args[i] = constructor_arg - i += 1 - - defaulted_args = [arg for arg in constructor_args if '=' in arg] - noarg_constructor = (not constructor_args or # empty arg list - # 'void' arg specifier - (len(constructor_args) == 1 and - constructor_args[0].strip() == 'void')) - onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg - not noarg_constructor) or - # all but at most one arg defaulted - (len(constructor_args) >= 1 and - not noarg_constructor and - len(defaulted_args) >= len(constructor_args) - 1)) - initializer_list_constructor = bool( - onearg_constructor and - Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) - copy_constructor = bool( - onearg_constructor and - Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' - % re.escape(base_classname), constructor_args[0].strip())) - - if (not is_marked_explicit and - onearg_constructor and - not initializer_list_constructor and - not copy_constructor): - if defaulted_args: - error(filename, linenum, 'runtime/explicit', 5, - 'Constructors callable with one argument ' - 'should be marked explicit.') - else: - error(filename, linenum, 'runtime/explicit', 5, - 'Single-parameter constructors should be marked explicit.') - elif is_marked_explicit and not onearg_constructor: - if noarg_constructor: - error(filename, linenum, 'runtime/explicit', 5, - 'Zero-parameter constructors should not be marked explicit.') - else: - error(filename, linenum, 'runtime/explicit', 0, - 'Constructors that require multiple arguments ' - 'should not be marked explicit.') + # Remove comments from the line, but leave in strings for now. + line = clean_lines.lines[linenum] + + if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): + error(filename, linenum, 'runtime/printf_format', 3, + '%q in format strings is deprecated. Use %ll instead.') + + if Search(r'printf\s*\(.*".*%\d+\$', line): + error(filename, linenum, 'runtime/printf_format', 2, + '%N$ formats are unconventional. Try rewriting to avoid them.') + + # Remove escaped backslashes before looking for undefined escapes. + line = line.replace('\\\\', '') + + if Search(r'("|\').*\\(%|\[|\(|{)', line): + error(filename, linenum, 'build/printf_format', 3, + '%, [, (, and { are undefined character escapes. Unescape them.') + + # For the rest, work with both comments and strings removed. + line = clean_lines.elided[linenum] + + if Search(r'\b(const|volatile|void|char|short|int|long' + r'|float|double|signed|unsigned' + r'|schar|u?int8|u?int16|u?int32|u?int64)' + r'\s+(register|static|extern|typedef)\b', line): + error(filename, linenum, 'build/storage_class', 5, + 'Storage class (static, extern, typedef, etc) should be first.') + + if Match(r'\s*#\s*endif\s*[^/\s]+', line): + error(filename, linenum, 'build/endif_comment', 5, + 'Uncommented text after #endif is non-standard. Use a comment.') + + if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): + error( + filename, linenum, 'build/forward_decl', 5, + 'Inner-style forward declarations are invalid. Remove this line.') + + if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', + line): + error( + filename, linenum, 'build/deprecated', 3, + '>? and ))?' + # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' + error(filename, linenum, 'runtime/member_string_references', 2, + 'const string& members are dangerous. It is much better to use ' + 'alternatives, such as pointers or simple constants.') + + # Everything else in this function operates on class declarations. + # Return early if the top of the nesting stack is not a class, or if + # the class head is not completed yet. + classinfo = nesting_state.InnermostClass() + if not classinfo or not classinfo.seen_open_brace: + return + + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] + + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. Also look for + # non-single-argument constructors which are also technically valid, but + # strongly suggest something is wrong. + explicit_constructor_match = Match( + r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*' + r'\(((?:[^()]|\([^()]*\))*)\)' % re.escape(base_classname), line) + + if explicit_constructor_match: + is_marked_explicit = explicit_constructor_match.group(1) + + if not explicit_constructor_match.group(2): + constructor_args = [] + else: + constructor_args = explicit_constructor_match.group(2).split(',') + + # collapse arguments so that commas in template parameter lists and function + # argument parameter lists don't split arguments in two + i = 0 + while i < len(constructor_args): + constructor_arg = constructor_args[i] + while (constructor_arg.count('<') > constructor_arg.count('>') or + constructor_arg.count('(') > constructor_arg.count(')')): + constructor_arg += ',' + constructor_args[i + 1] + del constructor_args[i + 1] + constructor_args[i] = constructor_arg + i += 1 + + defaulted_args = [arg for arg in constructor_args if '=' in arg] + noarg_constructor = ( + not constructor_args or # empty arg list + # 'void' arg specifier + (len(constructor_args) == 1 and + constructor_args[0].strip() == 'void')) + onearg_constructor = ( + ( + len(constructor_args) == 1 and # exactly one arg + not noarg_constructor) or + # all but at most one arg defaulted + (len(constructor_args) >= 1 and not noarg_constructor and + len(defaulted_args) >= len(constructor_args) - 1)) + initializer_list_constructor = bool( + onearg_constructor and + Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) + copy_constructor = bool( + onearg_constructor and + Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' % + re.escape(base_classname), constructor_args[0].strip())) + + if (not is_marked_explicit and onearg_constructor and + not initializer_list_constructor and not copy_constructor): + if defaulted_args: + error(filename, linenum, 'runtime/explicit', 5, + 'Constructors callable with one argument ' + 'should be marked explicit.') + else: + error( + filename, linenum, 'runtime/explicit', 5, + 'Single-parameter constructors should be marked explicit.') + elif is_marked_explicit and not onearg_constructor: + if noarg_constructor: + error( + filename, linenum, 'runtime/explicit', 5, + 'Zero-parameter constructors should not be marked explicit.') + else: + error(filename, linenum, 'runtime/explicit', 0, + 'Constructors that require multiple arguments ' + 'should not be marked explicit.') def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): - """Checks for the correctness of various spacing around function calls. + """Checks for the correctness of various spacing around function calls. Args: filename: The name of the current file. @@ -2752,75 +2771,74 @@ def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Since function calls often occur inside if/for/while/switch - # expressions - which have their own, more liberal conventions - we - # first see if we should be looking inside such an expression for a - # function call, to which we can apply more strict standards. - fncall = line # if there's no control flow construct, look at whole line - for pattern in (r'\bif\s*\((.*)\)\s*{', - r'\bfor\s*\((.*)\)\s*{', - r'\bwhile\s*\((.*)\)\s*[{;]', - r'\bswitch\s*\((.*)\)\s*{'): - match = Search(pattern, line) - if match: - fncall = match.group(1) # look inside the parens for function calls - break - - # Except in if/for/while/switch, there should never be space - # immediately inside parens (eg "f( 3, 4 )"). We make an exception - # for nested parens ( (a+b) + c ). Likewise, there should never be - # a space before a ( when it's a function argument. I assume it's a - # function argument when the char before the whitespace is legal in - # a function name (alnum + _) and we're not starting a macro. Also ignore - # pointers and references to arrays and functions coz they're too tricky: - # we use a very simple way to recognize these: - # " (something)(maybe-something)" or - # " (something)(maybe-something," or - # " (something)[something]" - # Note that we assume the contents of [] to be short enough that - # they'll never need to wrap. - if ( # Ignore control structures. - not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', - fncall) and - # Ignore pointers/references to functions. - not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and - # Ignore pointers/references to arrays. - not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): - if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space after ( in function call') - elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space after (') - if (Search(r'\w\s+\(', fncall) and - not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and - not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and - not Search(r'\bcase\s+\(', fncall)): - # TODO(unknown): Space after an operator function seem to be a common - # error, silence those for now by restricting them to highest verbosity. - if Search(r'\boperator_*\b', line): - error(filename, linenum, 'whitespace/parens', 0, - 'Extra space before ( in function call') - else: - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space before ( in function call') - # If the ) is followed only by a newline or a { + newline, assume it's - # part of a control statement (if/while/etc), and don't complain - if Search(r'[^)]\s+\)\s*[^{\s]', fncall): - # If the closing parenthesis is preceded by only whitespaces, - # try to give a more descriptive error message. - if Search(r'^\s+\)', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Closing ) should be moved to the previous line') - else: - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space before )') + line = clean_lines.elided[linenum] + + # Since function calls often occur inside if/for/while/switch + # expressions - which have their own, more liberal conventions - we + # first see if we should be looking inside such an expression for a + # function call, to which we can apply more strict standards. + fncall = line # if there's no control flow construct, look at whole line + for pattern in (r'\bif\s*\((.*)\)\s*{', r'\bfor\s*\((.*)\)\s*{', + r'\bwhile\s*\((.*)\)\s*[{;]', r'\bswitch\s*\((.*)\)\s*{'): + match = Search(pattern, line) + if match: + fncall = match.group(1) # look inside the parens for function calls + break + + # Except in if/for/while/switch, there should never be space + # immediately inside parens (eg "f( 3, 4 )"). We make an exception + # for nested parens ( (a+b) + c ). Likewise, there should never be + # a space before a ( when it's a function argument. I assume it's a + # function argument when the char before the whitespace is legal in + # a function name (alnum + _) and we're not starting a macro. Also ignore + # pointers and references to arrays and functions coz they're too tricky: + # we use a very simple way to recognize these: + # " (something)(maybe-something)" or + # " (something)(maybe-something," or + # " (something)[something]" + # Note that we assume the contents of [] to be short enough that + # they'll never need to wrap. + if ( # Ignore control structures. + not Search( + r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', + fncall) and + # Ignore pointers/references to functions. + not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and + # Ignore pointers/references to arrays. + not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): + if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space after ( in function call') + elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space after (') + if (Search(r'\w\s+\(', fncall) and + not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and + not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and + not Search(r'\bcase\s+\(', fncall)): + # TODO(unknown): Space after an operator function seem to be a common + # error, silence those for now by restricting them to highest verbosity. + if Search(r'\boperator_*\b', line): + error(filename, linenum, 'whitespace/parens', 0, + 'Extra space before ( in function call') + else: + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space before ( in function call') + # If the ) is followed only by a newline or a { + newline, assume it's + # part of a control statement (if/while/etc), and don't complain + if Search(r'[^)]\s+\)\s*[^{\s]', fncall): + # If the closing parenthesis is preceded by only whitespaces, + # try to give a more descriptive error message. + if Search(r'^\s+\)', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Closing ) should be moved to the previous line') + else: + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space before )') def IsBlankLine(line): - """Returns true if the given line is blank. + """Returns true if the given line is blank. We consider a line to be blank if the line is empty or consists of only white spaces. @@ -2831,26 +2849,26 @@ def IsBlankLine(line): Returns: True, if the given line is blank. """ - return not line or line.isspace() + return not line or line.isspace() def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, error): - is_namespace_indent_item = ( - len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and - nesting_state.previous_stack_top == nesting_state.stack[-2]) + is_namespace_indent_item = ( + len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and + nesting_state.previous_stack_top == nesting_state.stack[-2]) - if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, - clean_lines.elided, line): - CheckItemIndentationInNamespace(filename, clean_lines.elided, - line, error) + if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, + clean_lines.elided, line): + CheckItemIndentationInNamespace(filename, clean_lines.elided, line, + error) -def CheckForFunctionLengths(filename, clean_lines, linenum, - function_state, error): - """Reports for long function bodies. +def CheckForFunctionLengths(filename, clean_lines, linenum, function_state, + error): + """Reports for long function bodies. For an overview why this is done, see: http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions @@ -2871,56 +2889,57 @@ def CheckForFunctionLengths(filename, clean_lines, linenum, function_state: Current function name and lines in body so far. error: The function to call with any errors found. """ - lines = clean_lines.lines - line = lines[linenum] - joined_line = '' - - starting_func = False - regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... - match_result = Match(regexp, line) - if match_result: - # If the name is all caps and underscores, figure it's a macro and - # ignore it, unless it's TEST or TEST_F. - function_name = match_result.group(1).split()[-1] - if function_name == 'TEST' or function_name == 'TEST_F' or ( - not Match(r'[A-Z_]+$', function_name)): - starting_func = True - - if starting_func: - body_found = False - for start_linenum in xrange(linenum, clean_lines.NumLines()): - start_line = lines[start_linenum] - joined_line += ' ' + start_line.lstrip() - if Search(r'(;|})', start_line): # Declarations and trivial functions - body_found = True - break # ... ignore - elif Search(r'{', start_line): - body_found = True - function = Search(r'((\w|:)*)\(', line).group(1) - if Match(r'TEST', function): # Handle TEST... macros - parameter_regexp = Search(r'(\(.*\))', joined_line) - if parameter_regexp: # Ignore bad syntax - function += parameter_regexp.group(1) - else: - function += '()' - function_state.Begin(function) - break - if not body_found: - # No body for the function (or evidence of a non-function) was found. - error(filename, linenum, 'readability/fn_size', 5, - 'Lint failed to find start of function body.') - elif Match(r'^\}\s*$', line): # function end - function_state.Check(error, filename, linenum) - function_state.End() - elif not Match(r'^\s*$', line): - function_state.Count() # Count non-blank/non-comment lines. + lines = clean_lines.lines + line = lines[linenum] + joined_line = '' + + starting_func = False + regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... + match_result = Match(regexp, line) + if match_result: + # If the name is all caps and underscores, figure it's a macro and + # ignore it, unless it's TEST or TEST_F. + function_name = match_result.group(1).split()[-1] + if function_name == 'TEST' or function_name == 'TEST_F' or ( + not Match(r'[A-Z_]+$', function_name)): + starting_func = True + + if starting_func: + body_found = False + for start_linenum in xrange(linenum, clean_lines.NumLines()): + start_line = lines[start_linenum] + joined_line += ' ' + start_line.lstrip() + if Search(r'(;|})', + start_line): # Declarations and trivial functions + body_found = True + break # ... ignore + elif Search(r'{', start_line): + body_found = True + function = Search(r'((\w|:)*)\(', line).group(1) + if Match(r'TEST', function): # Handle TEST... macros + parameter_regexp = Search(r'(\(.*\))', joined_line) + if parameter_regexp: # Ignore bad syntax + function += parameter_regexp.group(1) + else: + function += '()' + function_state.Begin(function) + break + if not body_found: + # No body for the function (or evidence of a non-function) was found. + error(filename, linenum, 'readability/fn_size', 5, + 'Lint failed to find start of function body.') + elif Match(r'^\}\s*$', line): # function end + function_state.Check(error, filename, linenum) + function_state.End() + elif not Match(r'^\s*$', line): + function_state.Count() # Count non-blank/non-comment lines. _RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') def CheckComment(line, filename, linenum, next_line_start, error): - """Checks for common mistakes in comments. + """Checks for common mistakes in comments. Args: line: The line in question. @@ -2929,54 +2948,54 @@ def CheckComment(line, filename, linenum, next_line_start, error): next_line_start: The first non-whitespace column of the next line. error: The function to call with any errors found. """ - commentpos = line.find('//') - if commentpos != -1: - # Check if the // may be in quotes. If so, ignore it - # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison - if (line.count('"', 0, commentpos) - - line.count('\\"', 0, commentpos)) % 2 == 0: # not in quotes - # Allow one space for new scopes, two spaces otherwise: - if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and - ((commentpos >= 1 and - line[commentpos-1] not in string.whitespace) or - (commentpos >= 2 and - line[commentpos-2] not in string.whitespace))): - error(filename, linenum, 'whitespace/comments', 2, - 'At least two spaces is best between code and comments') - - # Checks for common mistakes in TODO comments. - comment = line[commentpos:] - match = _RE_PATTERN_TODO.match(comment) - if match: - # One whitespace is correct; zero whitespace is handled elsewhere. - leading_whitespace = match.group(1) - if len(leading_whitespace) > 1: - error(filename, linenum, 'whitespace/todo', 2, - 'Too many spaces before TODO') - - username = match.group(2) - if not username: - error(filename, linenum, 'readability/todo', 2, - 'Missing username in TODO; it should look like ' - '"// TODO(my_username): Stuff."') - - middle_whitespace = match.group(3) - # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison - if middle_whitespace != ' ' and middle_whitespace != '': - error(filename, linenum, 'whitespace/todo', 2, - 'TODO(my_username) should be followed by a space') - - # If the comment contains an alphanumeric character, there - # should be a space somewhere between it and the // unless - # it's a /// or //! Doxygen comment. - if (Match(r'//[^ ]*\w', comment) and - not Match(r'(///|//\!)(\s+|$)', comment)): - error(filename, linenum, 'whitespace/comments', 4, - 'Should have a space between // and comment') + commentpos = line.find('//') + if commentpos != -1: + # Check if the // may be in quotes. If so, ignore it + # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison + if (line.count('"', 0, commentpos) - line.count('\\"', 0, commentpos) + ) % 2 == 0: # not in quotes + # Allow one space for new scopes, two spaces otherwise: + if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) + and ((commentpos >= 1 and + line[commentpos - 1] not in string.whitespace) or + (commentpos >= 2 and + line[commentpos - 2] not in string.whitespace))): + error(filename, linenum, 'whitespace/comments', 2, + 'At least two spaces is best between code and comments') + + # Checks for common mistakes in TODO comments. + comment = line[commentpos:] + match = _RE_PATTERN_TODO.match(comment) + if match: + # One whitespace is correct; zero whitespace is handled elsewhere. + leading_whitespace = match.group(1) + if len(leading_whitespace) > 1: + error(filename, linenum, 'whitespace/todo', 2, + 'Too many spaces before TODO') + + username = match.group(2) + if not username: + error(filename, linenum, 'readability/todo', 2, + 'Missing username in TODO; it should look like ' + '"// TODO(my_username): Stuff."') + + middle_whitespace = match.group(3) + # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison + if middle_whitespace != ' ' and middle_whitespace != '': + error(filename, linenum, 'whitespace/todo', 2, + 'TODO(my_username) should be followed by a space') + + # If the comment contains an alphanumeric character, there + # should be a space somewhere between it and the // unless + # it's a /// or //! Doxygen comment. + if (Match(r'//[^ ]*\w', comment) and + not Match(r'(///|//\!)(\s+|$)', comment)): + error(filename, linenum, 'whitespace/comments', 4, + 'Should have a space between // and comment') def CheckAccess(filename, clean_lines, linenum, nesting_state, error): - """Checks for improper use of DISALLOW* macros. + """Checks for improper use of DISALLOW* macros. Args: filename: The name of the current file. @@ -2986,27 +3005,27 @@ def CheckAccess(filename, clean_lines, linenum, nesting_state, error): the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] # get rid of comments and strings - - matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' - r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) - if not matched: - return - if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): - if nesting_state.stack[-1].access != 'private': - error(filename, linenum, 'readability/constructors', 3, - '%s must be in the private: section' % matched.group(1)) - - else: - # Found DISALLOW* macro outside a class declaration, or perhaps it - # was used inside a function when it should have been part of the - # class declaration. We could issue a warning here, but it - # probably resulted in a compiler error already. - pass + line = clean_lines.elided[linenum] # get rid of comments and strings + + matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' + r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) + if not matched: + return + if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): + if nesting_state.stack[-1].access != 'private': + error(filename, linenum, 'readability/constructors', 3, + '%s must be in the private: section' % matched.group(1)) + + else: + # Found DISALLOW* macro outside a class declaration, or perhaps it + # was used inside a function when it should have been part of the + # class declaration. We could issue a warning here, but it + # probably resulted in a compiler error already. + pass def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): - """Checks for the correctness of various spacing issues in the code. + """Checks for the correctness of various spacing issues in the code. Things we check for: spaces around operators, spaces after if/for/while/switch, no spaces around parens in function calls, two @@ -3023,118 +3042,114 @@ def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): error: The function to call with any errors found. """ - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw = clean_lines.lines_without_raw_strings - line = raw[linenum] - - # Before nixing comments, check if the line is blank for no good - # reason. This includes the first line after a block is opened, and - # blank lines at the end of a function (ie, right before a line like '}' - # - # Skip all the blank line checks if we are immediately inside a - # namespace body. In other words, don't issue blank line warnings - # for this block: - # namespace { - # - # } - # - # A warning about missing end of namespace comments will be issued instead. - # - # Also skip blank line checks for 'extern "C"' blocks, which are formatted - # like namespaces. - if (IsBlankLine(line) and - not nesting_state.InNamespaceBody() and - not nesting_state.InExternC()): - elided = clean_lines.elided - prev_line = elided[linenum - 1] - prevbrace = prev_line.rfind('{') - # TODO(unknown): Don't complain if line before blank line, and line after, - # both start with alnums and are indented the same amount. - # This ignores whitespace at the start of a namespace block - # because those are not usually indented. - if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: - # OK, we have a blank line at the start of a code block. Before we - # complain, we check if it is an exception to the rule: The previous - # non-empty line has the parameters of a function header that are indented - # 4 spaces (because they did not fit in a 80 column line when placed on - # the same line as the function name). We also check for the case where - # the previous line is indented 6 spaces, which may happen when the - # initializers of a constructor do not fit into a 80 column line. - exception = False - if Match(r' {6}\w', prev_line): # Initializer list? - # We are looking for the opening column of initializer list, which - # should be indented 4 spaces to cause 6 space indentation afterwards. - search_position = linenum-2 - while (search_position >= 0 - and Match(r' {6}\w', elided[search_position])): - search_position -= 1 - exception = (search_position >= 0 - and elided[search_position][:5] == ' :') - else: - # Search for the function arguments or an initializer list. We use a - # simple heuristic here: If the line is indented 4 spaces; and we have a - # closing paren, without the opening paren, followed by an opening brace - # or colon (for initializer lists) we assume that it is the last line of - # a function header. If we have a colon indented 4 spaces, it is an - # initializer list. - exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', - prev_line) - or Match(r' {4}:', prev_line)) - - if not exception: - error(filename, linenum, 'whitespace/blank_line', 2, - 'Redundant blank line at the start of a code block ' - 'should be deleted.') - # Ignore blank lines at the end of a block in a long if-else - # chain, like this: - # if (condition1) { - # // Something followed by a blank line + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw = clean_lines.lines_without_raw_strings + line = raw[linenum] + + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}' + # + # Skip all the blank line checks if we are immediately inside a + # namespace body. In other words, don't issue blank line warnings + # for this block: + # namespace { # - # } else if (condition2) { - # // Something else # } + # + # A warning about missing end of namespace comments will be issued instead. + # + # Also skip blank line checks for 'extern "C"' blocks, which are formatted + # like namespaces. + if (IsBlankLine(line) and not nesting_state.InNamespaceBody() and + not nesting_state.InExternC()): + elided = clean_lines.elided + prev_line = elided[linenum - 1] + prevbrace = prev_line.rfind('{') + # TODO(unknown): Don't complain if line before blank line, and line after, + # both start with alnums and are indented the same amount. + # This ignores whitespace at the start of a namespace block + # because those are not usually indented. + if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the parameters of a function header that are indented + # 4 spaces (because they did not fit in a 80 column line when placed on + # the same line as the function name). We also check for the case where + # the previous line is indented 6 spaces, which may happen when the + # initializers of a constructor do not fit into a 80 column line. + exception = False + if Match(r' {6}\w', prev_line): # Initializer list? + # We are looking for the opening column of initializer list, which + # should be indented 4 spaces to cause 6 space indentation afterwards. + search_position = linenum - 2 + while (search_position >= 0 and + Match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 and + elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We use a + # simple heuristic here: If the line is indented 4 spaces; and we have a + # closing paren, without the opening paren, followed by an opening brace + # or colon (for initializer lists) we assume that it is the last line of + # a function header. If we have a colon indented 4 spaces, it is an + # initializer list. + exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + prev_line) or Match(r' {4}:', prev_line)) + + if not exception: + error(filename, linenum, 'whitespace/blank_line', 2, + 'Redundant blank line at the start of a code block ' + 'should be deleted.') + # Ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + if (next_line and Match(r'\s*}', next_line) and + next_line.find('} else ') == -1): + error(filename, linenum, 'whitespace/blank_line', 3, + 'Redundant blank line at the end of a code block ' + 'should be deleted.') + + matched = Match(r'\s*(public|protected|private):', prev_line) + if matched: + error(filename, linenum, 'whitespace/blank_line', 3, + 'Do not leave a blank line after "%s:"' % matched.group(1)) + + # Next, check comments + next_line_start = 0 if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - if (next_line - and Match(r'\s*}', next_line) - and next_line.find('} else ') == -1): - error(filename, linenum, 'whitespace/blank_line', 3, - 'Redundant blank line at the end of a code block ' - 'should be deleted.') - - matched = Match(r'\s*(public|protected|private):', prev_line) - if matched: - error(filename, linenum, 'whitespace/blank_line', 3, - 'Do not leave a blank line after "%s:"' % matched.group(1)) - - # Next, check comments - next_line_start = 0 - if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - next_line_start = len(next_line) - len(next_line.lstrip()) - CheckComment(line, filename, linenum, next_line_start, error) + next_line = raw[linenum + 1] + next_line_start = len(next_line) - len(next_line.lstrip()) + CheckComment(line, filename, linenum, next_line_start, error) - # get rid of comments and strings - line = clean_lines.elided[linenum] + # get rid of comments and strings + line = clean_lines.elided[linenum] - # You shouldn't have spaces before your brackets, except maybe after - # 'delete []' or 'return []() {};' - if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Extra space before [') + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []' or 'return []() {};' + if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line): + error(filename, linenum, 'whitespace/braces', 5, 'Extra space before [') - # In range-based for, we wanted spaces before and after the colon, but - # not around "::" tokens that might appear. - if (Search(r'for *\(.*[^:]:[^: ]', line) or - Search(r'for *\(.*[^: ]:[^:]', line)): - error(filename, linenum, 'whitespace/forcolon', 2, - 'Missing space around colon in range-based for loop') + # In range-based for, we wanted spaces before and after the colon, but + # not around "::" tokens that might appear. + if (Search(r'for *\(.*[^:]:[^: ]', line) or + Search(r'for *\(.*[^: ]:[^:]', line)): + error(filename, linenum, 'whitespace/forcolon', 2, + 'Missing space around colon in range-based for loop') def CheckOperatorSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around operators. + """Checks for horizontal spacing around operators. Args: filename: The name of the current file. @@ -3142,114 +3157,116 @@ def CheckOperatorSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Don't try to do spacing checks for operator methods. Do this by - # replacing the troublesome characters with something else, - # preserving column position for all other characters. - # - # The replacement is done repeatedly to avoid false positives from - # operators that call operators. - while True: - match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) - if match: - line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) - else: - break - - # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". - # Otherwise not. Note we only check for non-spaces on *both* sides; - # sometimes people put non-spaces on one side when aligning ='s among - # many lines (not that this is behavior that I approve of...) - if ((Search(r'[\w.]=', line) or - Search(r'=[\w.]', line)) - and not Search(r'\b(if|while|for) ', line) - # Operators taken from [lex.operators] in C++11 standard. - and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) - and not Search(r'operator=', line)): - error(filename, linenum, 'whitespace/operators', 4, - 'Missing spaces around =') - - # It's ok not to have spaces around binary operators like + - * /, but if - # there's too little whitespace, we get concerned. It's hard to tell, - # though, so we punt on this one for now. TODO. - - # You should always have whitespace around binary operators. - # - # Check <= and >= first to avoid false positives with < and >, then - # check non-include lines for spacing around < and >. - # - # If the operator is followed by a comma, assume it's be used in a - # macro context and don't do any checks. This avoids false - # positives. - # - # Note that && is not included here. Those are checked separately - # in CheckRValueReference - match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around %s' % match.group(1)) - elif not Match(r'#.*include', line): - # Look for < that is not surrounded by spaces. This is only - # triggered if both sides are missing spaces, even though - # technically should should flag if at least one side is missing a - # space. This is done to avoid some false positives with shifts. - match = Match(r'^(.*[^\s<])<[^\s=<,]', line) + line = clean_lines.elided[linenum] + + # Don't try to do spacing checks for operator methods. Do this by + # replacing the troublesome characters with something else, + # preserving column position for all other characters. + # + # The replacement is done repeatedly to avoid false positives from + # operators that call operators. + while True: + match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) + if match: + line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) + else: + break + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if ((Search(r'[\w.]=', line) or + Search(r'=[\w.]', line)) and not Search(r'\b(if|while|for) ', line) + # Operators taken from [lex.operators] in C++11 standard. + and + not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) and + not Search(r'operator=', line)): + error(filename, linenum, 'whitespace/operators', 4, + 'Missing spaces around =') + + # It's ok not to have spaces around binary operators like + - * /, but if + # there's too little whitespace, we get concerned. It's hard to tell, + # though, so we punt on this one for now. TODO. + + # You should always have whitespace around binary operators. + # + # Check <= and >= first to avoid false positives with < and >, then + # check non-include lines for spacing around < and >. + # + # If the operator is followed by a comma, assume it's be used in a + # macro context and don't do any checks. This avoids false + # positives. + # + # Note that && is not included here. Those are checked separately + # in CheckRValueReference + match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) if match: - (_, _, end_pos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if end_pos <= -1: error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <') + 'Missing spaces around %s' % match.group(1)) + elif not Match(r'#.*include', line): + # Look for < that is not surrounded by spaces. This is only + # triggered if both sides are missing spaces, even though + # technically should should flag if at least one side is missing a + # space. This is done to avoid some false positives with shifts. + match = Match(r'^(.*[^\s<])<[^\s=<,]', line) + if match: + (_, _, end_pos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if end_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <') + + # Look for > that is not surrounded by spaces. Similar to the + # above, we only trigger if both sides are missing spaces to avoid + # false positives with shifts. + match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) + if match: + (_, _, start_pos) = ReverseCloseExpression(clean_lines, linenum, + len(match.group(1))) + if start_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >') + + # We allow no-spaces around << when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + # + # We also allow operators following an opening parenthesis, since + # those tend to be macros that deal with operators. + match = Search(r'(operator|[^\s(<])(?:L|UL|ULL|l|ul|ull)?<<([^\s,=<])', + line) + if (match and + not (match.group(1).isdigit() and match.group(2).isdigit()) and + not (match.group(1) == 'operator' and match.group(2) == ';')): + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <<') - # Look for > that is not surrounded by spaces. Similar to the - # above, we only trigger if both sides are missing spaces to avoid - # false positives with shifts. - match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) + # We allow no-spaces around >> for almost anything. This is because + # C++11 allows ">>" to close nested templates, which accounts for + # most cases when ">>" is not followed by a space. + # + # We still warn on ">>" followed by alpha character, because that is + # likely due to ">>" being used for right shifts, e.g.: + # value >> alpha + # + # When ">>" is used to close templates, the alphanumeric letter that + # follows would be part of an identifier, and there should still be + # a space separating the template type and the identifier. + # type> alpha + match = Search(r'>>[a-zA-Z_]', line) if match: - (_, _, start_pos) = ReverseCloseExpression( - clean_lines, linenum, len(match.group(1))) - if start_pos <= -1: error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >') - - # We allow no-spaces around << when used like this: 10<<20, but - # not otherwise (particularly, not when used as streams) - # - # We also allow operators following an opening parenthesis, since - # those tend to be macros that deal with operators. - match = Search(r'(operator|[^\s(<])(?:L|UL|ULL|l|ul|ull)?<<([^\s,=<])', line) - if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and - not (match.group(1) == 'operator' and match.group(2) == ';')): - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <<') - - # We allow no-spaces around >> for almost anything. This is because - # C++11 allows ">>" to close nested templates, which accounts for - # most cases when ">>" is not followed by a space. - # - # We still warn on ">>" followed by alpha character, because that is - # likely due to ">>" being used for right shifts, e.g.: - # value >> alpha - # - # When ">>" is used to close templates, the alphanumeric letter that - # follows would be part of an identifier, and there should still be - # a space separating the template type and the identifier. - # type> alpha - match = Search(r'>>[a-zA-Z_]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >>') - - # There shouldn't be space around unary operators - match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) - if match: - error(filename, linenum, 'whitespace/operators', 4, - 'Extra space for operator %s' % match.group(1)) + 'Missing spaces around >>') + + # There shouldn't be space around unary operators + match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if match: + error(filename, linenum, 'whitespace/operators', 4, + 'Extra space for operator %s' % match.group(1)) def CheckParenthesisSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around parentheses. + """Checks for horizontal spacing around parentheses. Args: filename: The name of the current file. @@ -3257,37 +3274,36 @@ def CheckParenthesisSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # No spaces after an if, while, switch, or for - match = Search(r' (if\(|for\(|while\(|switch\()', line) - if match: - error(filename, linenum, 'whitespace/parens', 5, - 'Missing space before ( in %s' % match.group(1)) - - # For if/for/while/switch, the left and right parens should be - # consistent about how many spaces are inside the parens, and - # there should either be zero or one spaces inside the parens. - # We don't want: "if ( foo)" or "if ( foo )". - # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. - match = Search(r'\b(if|for|while|switch)\s*' - r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', - line) - if match: - if len(match.group(2)) != len(match.group(4)): - if not (match.group(3) == ';' and - len(match.group(2)) == 1 + len(match.group(4)) or - not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + line = clean_lines.elided[linenum] + + # No spaces after an if, while, switch, or for + match = Search(r' (if\(|for\(|while\(|switch\()', line) + if match: error(filename, linenum, 'whitespace/parens', 5, - 'Mismatching spaces inside () in %s' % match.group(1)) - if len(match.group(2)) not in [0, 1]: - error(filename, linenum, 'whitespace/parens', 5, - 'Should have zero or one spaces inside ( and ) in %s' % - match.group(1)) + 'Missing space before ( in %s' % match.group(1)) + + # For if/for/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + match = Search(r'\b(if|for|while|switch)\s*' + r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', line) + if match: + if len(match.group(2)) != len(match.group(4)): + if not (match.group(3) == ';' and + len(match.group(2)) == 1 + len(match.group(4)) or + not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + error(filename, linenum, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % match.group(1)) + if len(match.group(2)) not in [0, 1]: + error(filename, linenum, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + match.group(1)) def CheckCommaSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing near commas and semicolons. + """Checks for horizontal spacing near commas and semicolons. Args: filename: The name of the current file. @@ -3295,35 +3311,34 @@ def CheckCommaSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - raw = clean_lines.lines_without_raw_strings - line = clean_lines.elided[linenum] - - # You should always have a space after a comma (either as fn arg or operator) - # - # This does not apply when the non-space character following the - # comma is another comma, since the only time when that happens is - # for empty macro arguments. - # - # We run this check in two passes: first pass on elided lines to - # verify that lines contain missing whitespaces, second pass on raw - # lines to confirm that those missing whitespaces are not due to - # elided comments. - if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and - Search(r',[^,\s]', raw[linenum])): - error(filename, linenum, 'whitespace/comma', 3, - 'Missing space after ,') - - # You should always have a space after a semicolon - # except for few corner cases - # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more - # space after ; - if Search(r';[^\s};\\)/]', line): - error(filename, linenum, 'whitespace/semicolon', 3, - 'Missing space after ;') + raw = clean_lines.lines_without_raw_strings + line = clean_lines.elided[linenum] + + # You should always have a space after a comma (either as fn arg or operator) + # + # This does not apply when the non-space character following the + # comma is another comma, since the only time when that happens is + # for empty macro arguments. + # + # We run this check in two passes: first pass on elided lines to + # verify that lines contain missing whitespaces, second pass on raw + # lines to confirm that those missing whitespaces are not due to + # elided comments. + if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and + Search(r',[^,\s]', raw[linenum])): + error(filename, linenum, 'whitespace/comma', 3, 'Missing space after ,') + + # You should always have a space after a semicolon + # except for few corner cases + # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more + # space after ; + if Search(r';[^\s};\\)/]', line): + error(filename, linenum, 'whitespace/semicolon', 3, + 'Missing space after ;') def CheckBracesSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing near commas. + """Checks for horizontal spacing near commas. Args: filename: The name of the current file. @@ -3331,78 +3346,78 @@ def CheckBracesSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Except after an opening paren, or after another opening brace (in case of - # an initializer list, for instance), you should have spaces before your - # braces. And since you should never have braces at the beginning of a line, - # this is an easy test. - match = Match(r'^(.*[^ ({>]){', line) - if match: - # Try a bit harder to check for brace initialization. This - # happens in one of the following forms: - # Constructor() : initializer_list_{} { ... } - # Constructor{}.MemberFunction() - # Type variable{}; - # FunctionCall(type{}, ...); - # LastArgument(..., type{}); - # LOG(INFO) << type{} << " ..."; - # map_of_type[{...}] = ...; - # ternary = expr ? new type{} : nullptr; - # OuterTemplate{}> - # - # We check for the character following the closing brace, and - # silence the warning if it's one of those listed above, i.e. - # "{.;,)<>]:". - # - # To account for nested initializer list, we allow any number of - # closing braces up to "{;,)<". We can't simply silence the - # warning on first sight of closing brace, because that would - # cause false negatives for things that are not initializer lists. - # Silence this: But not this: - # Outer{ if (...) { - # Inner{...} if (...){ // Missing space before { - # }; } - # - # There is a false negative with this approach if people inserted - # spurious semicolons, e.g. "if (cond){};", but we will catch the - # spurious semicolon with a separate check. - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - trailing_text = '' - if endpos > -1: - trailing_text = endline[endpos:] - for offset in xrange(endlinenum + 1, - min(endlinenum + 3, clean_lines.NumLines() - 1)): - trailing_text += clean_lines.elided[offset] - if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before {') - - # Make sure '} else {' has spaces. - if Search(r'}else', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before else') - - # You shouldn't have a space before a semicolon at the end of the line. - # There's a special case for "for" since the style guide allows space before - # the semicolon there. - if Search(r':\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Semicolon defining empty statement. Use {} instead.') - elif Search(r'^\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Line contains only semicolon. If this should be an empty statement, ' - 'use {} instead.') - elif (Search(r'\s+;\s*$', line) and - not Search(r'\bfor\b', line)): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Extra space before last semicolon. If this should be an empty ' - 'statement, use {} instead.') + line = clean_lines.elided[linenum] + + # Except after an opening paren, or after another opening brace (in case of + # an initializer list, for instance), you should have spaces before your + # braces. And since you should never have braces at the beginning of a line, + # this is an easy test. + match = Match(r'^(.*[^ ({>]){', line) + if match: + # Try a bit harder to check for brace initialization. This + # happens in one of the following forms: + # Constructor() : initializer_list_{} { ... } + # Constructor{}.MemberFunction() + # Type variable{}; + # FunctionCall(type{}, ...); + # LastArgument(..., type{}); + # LOG(INFO) << type{} << " ..."; + # map_of_type[{...}] = ...; + # ternary = expr ? new type{} : nullptr; + # OuterTemplate{}> + # + # We check for the character following the closing brace, and + # silence the warning if it's one of those listed above, i.e. + # "{.;,)<>]:". + # + # To account for nested initializer list, we allow any number of + # closing braces up to "{;,)<". We can't simply silence the + # warning on first sight of closing brace, because that would + # cause false negatives for things that are not initializer lists. + # Silence this: But not this: + # Outer{ if (...) { + # Inner{...} if (...){ // Missing space before { + # }; } + # + # There is a false negative with this approach if people inserted + # spurious semicolons, e.g. "if (cond){};", but we will catch the + # spurious semicolon with a separate check. + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + trailing_text = '' + if endpos > -1: + trailing_text = endline[endpos:] + for offset in xrange(endlinenum + 1, + min(endlinenum + 3, clean_lines.NumLines() - 1)): + trailing_text += clean_lines.elided[offset] + if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before {') + + # Make sure '} else {' has spaces. + if Search(r'}else', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before else') + + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if Search(r':\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use {} instead.') + elif Search(r'^\s*;\s*$', line): + error( + filename, linenum, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use {} instead.') + elif (Search(r'\s+;\s*$', line) and not Search(r'\bfor\b', line)): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use {} instead.') def IsDecltype(clean_lines, linenum, column): - """Check if the token ending on (linenum, column) is decltype(). + """Check if the token ending on (linenum, column) is decltype(). Args: clean_lines: A CleansedLines instance containing the file. @@ -3411,16 +3426,16 @@ def IsDecltype(clean_lines, linenum, column): Returns: True if this token is decltype() expression, False otherwise. """ - (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) - if start_col < 0: + (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) + if start_col < 0: + return False + if Search(r'\bdecltype\s*$', text[0:start_col]): + return True return False - if Search(r'\bdecltype\s*$', text[0:start_col]): - return True - return False def IsTemplateParameterList(clean_lines, linenum, column): - """Check if the token ending on (linenum, column) is the end of template<>. + """Check if the token ending on (linenum, column) is the end of template<>. Args: clean_lines: A CleansedLines instance containing the file. @@ -3429,16 +3444,16 @@ def IsTemplateParameterList(clean_lines, linenum, column): Returns: True if this token is end of a template parameter list, False otherwise. """ - (_, startline, startpos) = ReverseCloseExpression( - clean_lines, linenum, column) - if (startpos > -1 and - Search(r'\btemplate\s*$', clean_lines.elided[startline][0:startpos])): - return True - return False + (_, startline, startpos) = ReverseCloseExpression(clean_lines, linenum, + column) + if (startpos > -1 and Search(r'\btemplate\s*$', + clean_lines.elided[startline][0:startpos])): + return True + return False def IsRValueType(typenames, clean_lines, nesting_state, linenum, column): - """Check if the token ending on (linenum, column) is a type. + """Check if the token ending on (linenum, column) is a type. Assumes that text to the right of the column is "&&" or a function name. @@ -3453,196 +3468,198 @@ def IsRValueType(typenames, clean_lines, nesting_state, linenum, column): Returns: True if this token is a type, False if we are not sure. """ - prefix = clean_lines.elided[linenum][0:column] - - # Get one word to the left. If we failed to do so, this is most - # likely not a type, since it's unlikely that the type name and "&&" - # would be split across multiple lines. - match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix) - if not match: - return False + prefix = clean_lines.elided[linenum][0:column] - # Check text following the token. If it's "&&>" or "&&," or "&&...", it's - # most likely a rvalue reference used inside a template. - suffix = clean_lines.elided[linenum][column:] - if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix): - return True + # Get one word to the left. If we failed to do so, this is most + # likely not a type, since it's unlikely that the type name and "&&" + # would be split across multiple lines. + match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix) + if not match: + return False - # Check for known types and end of templates: - # int&& variable - # vector&& variable - # - # Because this function is called recursively, we also need to - # recognize pointer and reference types: - # int* Function() - # int& Function() - if (match.group(2) in typenames or - match.group(2) in ['char', 'char16_t', 'char32_t', 'wchar_t', 'bool', - 'short', 'int', 'long', 'signed', 'unsigned', - 'float', 'double', 'void', 'auto', '>', '*', '&']): - return True + # Check text following the token. If it's "&&>" or "&&," or "&&...", it's + # most likely a rvalue reference used inside a template. + suffix = clean_lines.elided[linenum][column:] + if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix): + return True - # If we see a close parenthesis, look for decltype on the other side. - # decltype would unambiguously identify a type, anything else is - # probably a parenthesized expression and not a type. - if match.group(2) == ')': - return IsDecltype( - clean_lines, linenum, len(match.group(1)) + len(match.group(2)) - 1) - - # Check for casts and cv-qualifiers. - # match.group(1) remainder - # -------------- --------- - # const_cast< type&& - # const type&& - # type const&& - if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|' - r'reinterpret_cast\s*<|\w+\s)\s*$', - match.group(1)): - return True + # Check for known types and end of templates: + # int&& variable + # vector&& variable + # + # Because this function is called recursively, we also need to + # recognize pointer and reference types: + # int* Function() + # int& Function() + if (match.group(2) in typenames or match.group(2) in [ + 'char', 'char16_t', 'char32_t', 'wchar_t', 'bool', 'short', 'int', + 'long', 'signed', 'unsigned', 'float', 'double', 'void', 'auto', + '>', '*', '&' + ]): + return True - # Look for a preceding symbol that might help differentiate the context. - # These are the cases that would be ambiguous: - # match.group(1) remainder - # -------------- --------- - # Call ( expression && - # Declaration ( type&& - # sizeof ( type&& - # if ( expression && - # while ( expression && - # for ( type&& - # for( ; expression && - # statement ; type&& - # block { type&& - # constructor { expression && - start = linenum - line = match.group(1) - match_symbol = None - while start >= 0: - # We want to skip over identifiers and commas to get to a symbol. - # Commas are skipped so that we can find the opening parenthesis - # for function parameter lists. - match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line) - if match_symbol: - break - start -= 1 - line = clean_lines.elided[start] - - if not match_symbol: - # Probably the first statement in the file is an rvalue reference - return True + # If we see a close parenthesis, look for decltype on the other side. + # decltype would unambiguously identify a type, anything else is + # probably a parenthesized expression and not a type. + if match.group(2) == ')': + return IsDecltype(clean_lines, linenum, + len(match.group(1)) + len(match.group(2)) - 1) + + # Check for casts and cv-qualifiers. + # match.group(1) remainder + # -------------- --------- + # const_cast< type&& + # const type&& + # type const&& + if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|' + r'reinterpret_cast\s*<|\w+\s)\s*$', match.group(1)): + return True - if match_symbol.group(2) == '}': - # Found closing brace, probably an indicate of this: - # block{} type&& - return True + # Look for a preceding symbol that might help differentiate the context. + # These are the cases that would be ambiguous: + # match.group(1) remainder + # -------------- --------- + # Call ( expression && + # Declaration ( type&& + # sizeof ( type&& + # if ( expression && + # while ( expression && + # for ( type&& + # for( ; expression && + # statement ; type&& + # block { type&& + # constructor { expression && + start = linenum + line = match.group(1) + match_symbol = None + while start >= 0: + # We want to skip over identifiers and commas to get to a symbol. + # Commas are skipped so that we can find the opening parenthesis + # for function parameter lists. + match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line) + if match_symbol: + break + start -= 1 + line = clean_lines.elided[start] - if match_symbol.group(2) == ';': - # Found semicolon, probably one of these: - # for(; expression && - # statement; type&& - - # Look for the previous 'for(' in the previous lines. - before_text = match_symbol.group(1) - for i in xrange(start - 1, max(start - 6, 0), -1): - before_text = clean_lines.elided[i] + before_text - if Search(r'for\s*\([^{};]*$', before_text): - # This is the condition inside a for-loop - return False - - # Did not find a for-init-statement before this semicolon, so this - # is probably a new statement and not a condition. - return True + if not match_symbol: + # Probably the first statement in the file is an rvalue reference + return True - if match_symbol.group(2) == '{': - # Found opening brace, probably one of these: - # block{ type&& = ... ; } - # constructor{ expression && expression } + if match_symbol.group(2) == '}': + # Found closing brace, probably an indicate of this: + # block{} type&& + return True - # Look for a closing brace or a semicolon. If we see a semicolon - # first, this is probably a rvalue reference. - line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1] - end = start - depth = 1 - while True: - for ch in line: - if ch == ';': - return True - elif ch == '{': - depth += 1 - elif ch == '}': - depth -= 1 - if depth == 0: + if match_symbol.group(2) == ';': + # Found semicolon, probably one of these: + # for(; expression && + # statement; type&& + + # Look for the previous 'for(' in the previous lines. + before_text = match_symbol.group(1) + for i in xrange(start - 1, max(start - 6, 0), -1): + before_text = clean_lines.elided[i] + before_text + if Search(r'for\s*\([^{};]*$', before_text): + # This is the condition inside a for-loop return False - end += 1 - if end >= clean_lines.NumLines(): - break - line = clean_lines.elided[end] - # Incomplete program? - return False - if match_symbol.group(2) == '(': - # Opening parenthesis. Need to check what's to the left of the - # parenthesis. Look back one extra line for additional context. - before_text = match_symbol.group(1) - if linenum > 1: - before_text = clean_lines.elided[linenum - 1] + before_text - before_text = match_symbol.group(1) - - # Patterns that are likely to be types: - # [](type&& - # for (type&& - # sizeof(type&& - # operator=(type&& - # - if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', before_text): - return True - - # Patterns that are likely to be expressions: - # if (expression && - # while (expression && - # : initializer(expression && - # , initializer(expression && - # ( FunctionCall(expression && - # + FunctionCall(expression && - # + (expression && - # - # The last '+' represents operators such as '+' and '-'. - if Search(r'(?:\bif|\bwhile|[-+=%^(]*>)?\s*$', - match_symbol.group(1)) - if match_func: - # Check for constructors, which don't have return types. - if Search(r'\b(?:explicit|inline)$', match_func.group(1)): - return True - implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', prefix) - if (implicit_constructor and - implicit_constructor.group(1) == implicit_constructor.group(2)): + # Did not find a for-init-statement before this semicolon, so this + # is probably a new statement and not a condition. return True - return IsRValueType(typenames, clean_lines, nesting_state, linenum, - len(match_func.group(1))) - # Nothing before the function name. If this is inside a block scope, - # this is probably a function call. - return not (nesting_state.previous_stack_top and - nesting_state.previous_stack_top.IsBlockInfo()) + if match_symbol.group(2) == '{': + # Found opening brace, probably one of these: + # block{ type&& = ... ; } + # constructor{ expression && expression } + + # Look for a closing brace or a semicolon. If we see a semicolon + # first, this is probably a rvalue reference. + line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1] + end = start + depth = 1 + while True: + for ch in line: + if ch == ';': + return True + elif ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + if depth == 0: + return False + end += 1 + if end >= clean_lines.NumLines(): + break + line = clean_lines.elided[end] + # Incomplete program? + return False - if match_symbol.group(2) == '>': - # Possibly a closing bracket, check that what's on the other side - # looks like the start of a template. - return IsTemplateParameterList( - clean_lines, start, len(match_symbol.group(1))) + if match_symbol.group(2) == '(': + # Opening parenthesis. Need to check what's to the left of the + # parenthesis. Look back one extra line for additional context. + before_text = match_symbol.group(1) + if linenum > 1: + before_text = clean_lines.elided[linenum - 1] + before_text + before_text = match_symbol.group(1) + + # Patterns that are likely to be types: + # [](type&& + # for (type&& + # sizeof(type&& + # operator=(type&& + # + if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', + before_text): + return True + + # Patterns that are likely to be expressions: + # if (expression && + # while (expression && + # : initializer(expression && + # , initializer(expression && + # ( FunctionCall(expression && + # + FunctionCall(expression && + # + (expression && + # + # The last '+' represents operators such as '+' and '-'. + if Search(r'(?:\bif|\bwhile|[-+=%^(]*>)?\s*$', + match_symbol.group(1)) + if match_func: + # Check for constructors, which don't have return types. + if Search(r'\b(?:explicit|inline)$', match_func.group(1)): + return True + implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', + prefix) + if (implicit_constructor and implicit_constructor.group(1) == + implicit_constructor.group(2)): + return True + return IsRValueType(typenames, clean_lines, nesting_state, linenum, + len(match_func.group(1))) + + # Nothing before the function name. If this is inside a block scope, + # this is probably a function call. + return not (nesting_state.previous_stack_top and + nesting_state.previous_stack_top.IsBlockInfo()) + + if match_symbol.group(2) == '>': + # Possibly a closing bracket, check that what's on the other side + # looks like the start of a template. + return IsTemplateParameterList(clean_lines, start, + len(match_symbol.group(1))) + + # Some other symbol, usually something like "a=b&&c". This is most + # likely not a type. + return False def IsDeletedOrDefault(clean_lines, linenum): - """Check if current constructor or operator is deleted or default. + """Check if current constructor or operator is deleted or default. Args: clean_lines: A CleansedLines instance containing the file. @@ -3650,18 +3667,18 @@ def IsDeletedOrDefault(clean_lines, linenum): Returns: True if this is a deleted or default constructor. """ - open_paren = clean_lines.elided[linenum].find('(') - if open_paren < 0: - return False - (close_line, _, close_paren) = CloseExpression( - clean_lines, linenum, open_paren) - if close_paren < 0: - return False - return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:]) + open_paren = clean_lines.elided[linenum].find('(') + if open_paren < 0: + return False + (close_line, _, close_paren) = CloseExpression(clean_lines, linenum, + open_paren) + if close_paren < 0: + return False + return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:]) def IsRValueAllowed(clean_lines, linenum, typenames): - """Check if RValue reference is allowed on a particular line. + """Check if RValue reference is allowed on a particular line. Args: clean_lines: A CleansedLines instance containing the file. @@ -3670,56 +3687,57 @@ def IsRValueAllowed(clean_lines, linenum, typenames): Returns: True if line is within the region where RValue references are allowed. """ - # Allow region marked by PUSH/POP macros - for i in xrange(linenum, 0, -1): - line = clean_lines.elided[i] - if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): - if not line.endswith('PUSH'): - return False - for j in xrange(linenum, clean_lines.NumLines(), 1): - line = clean_lines.elided[j] + # Allow region marked by PUSH/POP macros + for i in xrange(linenum, 0, -1): + line = clean_lines.elided[i] if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): - return line.endswith('POP') - - # Allow operator= - line = clean_lines.elided[linenum] - if Search(r'\boperator\s*=\s*\(', line): - return IsDeletedOrDefault(clean_lines, linenum) - - # Allow constructors - match = Match(r'\s*(?:[\w<>]+::)*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line) - if match and match.group(1) == match.group(2): - return IsDeletedOrDefault(clean_lines, linenum) - if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line): - return IsDeletedOrDefault(clean_lines, linenum) - - if Match(r'\s*[\w<>]+\s*\(', line): - previous_line = 'ReturnType' - if linenum > 0: - previous_line = clean_lines.elided[linenum - 1] - if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', previous_line): - return IsDeletedOrDefault(clean_lines, linenum) + if not line.endswith('PUSH'): + return False + for j in xrange(linenum, clean_lines.NumLines(), 1): + line = clean_lines.elided[j] + if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): + return line.endswith('POP') - # Reject types not mentioned in template-argument-list - while line: - match = Match(r'^.*?(\w+)\s*&&(.*)$', line) - if not match: - break - if match.group(1) not in typenames: - return False - line = match.group(2) + # Allow operator= + line = clean_lines.elided[linenum] + if Search(r'\boperator\s*=\s*\(', line): + return IsDeletedOrDefault(clean_lines, linenum) + + # Allow constructors + match = Match(r'\s*(?:[\w<>]+::)*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line) + if match and match.group(1) == match.group(2): + return IsDeletedOrDefault(clean_lines, linenum) + if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line): + return IsDeletedOrDefault(clean_lines, linenum) + + if Match(r'\s*[\w<>]+\s*\(', line): + previous_line = 'ReturnType' + if linenum > 0: + previous_line = clean_lines.elided[linenum - 1] + if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', + previous_line): + return IsDeletedOrDefault(clean_lines, linenum) + + # Reject types not mentioned in template-argument-list + while line: + match = Match(r'^.*?(\w+)\s*&&(.*)$', line) + if not match: + break + if match.group(1) not in typenames: + return False + line = match.group(2) - # All RValue types that were in template-argument-list should have - # been removed by now. Those were allowed, assuming that they will - # be forwarded. - # - # If there are no remaining RValue types left (i.e. types that were - # not found in template-argument-list), flag those as not allowed. - return line.find('&&') < 0 + # All RValue types that were in template-argument-list should have + # been removed by now. Those were allowed, assuming that they will + # be forwarded. + # + # If there are no remaining RValue types left (i.e. types that were + # not found in template-argument-list), flag those as not allowed. + return line.find('&&') < 0 def GetTemplateArgs(clean_lines, linenum): - """Find list of template arguments associated with this function declaration. + """Find list of template arguments associated with this function declaration. Args: clean_lines: A CleansedLines instance containing the file. @@ -3729,61 +3747,63 @@ def GetTemplateArgs(clean_lines, linenum): Set of type names, or empty set if this does not appear to have any template parameters. """ - # Find start of function - func_line = linenum - while func_line > 0: - line = clean_lines.elided[func_line] - if Match(r'^\s*$', line): - return set() - if line.find('(') >= 0: - break - func_line -= 1 - if func_line == 0: - return set() - - # Collapse template-argument-list into a single string - argument_list = '' - match = Match(r'^(\s*template\s*)<', clean_lines.elided[func_line]) - if match: - # template-argument-list on the same line as function name - start_col = len(match.group(1)) - _, end_line, end_col = CloseExpression(clean_lines, func_line, start_col) - if end_col > -1 and end_line == func_line: - start_col += 1 # Skip the opening bracket - argument_list = clean_lines.elided[func_line][start_col:end_col] - - elif func_line > 1: - # template-argument-list one line before function name - match = Match(r'^(.*)>\s*$', clean_lines.elided[func_line - 1]) + # Find start of function + func_line = linenum + while func_line > 0: + line = clean_lines.elided[func_line] + if Match(r'^\s*$', line): + return set() + if line.find('(') >= 0: + break + func_line -= 1 + if func_line == 0: + return set() + + # Collapse template-argument-list into a single string + argument_list = '' + match = Match(r'^(\s*template\s*)<', clean_lines.elided[func_line]) if match: - end_col = len(match.group(1)) - _, start_line, start_col = ReverseCloseExpression( - clean_lines, func_line - 1, end_col) - if start_col > -1: - start_col += 1 # Skip the opening bracket - while start_line < func_line - 1: - argument_list += clean_lines.elided[start_line][start_col:] - start_col = 0 - start_line += 1 - argument_list += clean_lines.elided[func_line - 1][start_col:end_col] - - if not argument_list: - return set() - - # Extract type names - typenames = set() - while True: - match = Match(r'^[,\s]*(?:typename|class)(?:\.\.\.)?\s+(\w+)(.*)$', - argument_list) - if not match: - break - typenames.add(match.group(1)) - argument_list = match.group(2) - return typenames + # template-argument-list on the same line as function name + start_col = len(match.group(1)) + _, end_line, end_col = CloseExpression(clean_lines, func_line, + start_col) + if end_col > -1 and end_line == func_line: + start_col += 1 # Skip the opening bracket + argument_list = clean_lines.elided[func_line][start_col:end_col] + + elif func_line > 1: + # template-argument-list one line before function name + match = Match(r'^(.*)>\s*$', clean_lines.elided[func_line - 1]) + if match: + end_col = len(match.group(1)) + _, start_line, start_col = ReverseCloseExpression( + clean_lines, func_line - 1, end_col) + if start_col > -1: + start_col += 1 # Skip the opening bracket + while start_line < func_line - 1: + argument_list += clean_lines.elided[start_line][start_col:] + start_col = 0 + start_line += 1 + argument_list += clean_lines.elided[func_line - 1][start_col: + end_col] + + if not argument_list: + return set() + + # Extract type names + typenames = set() + while True: + match = Match(r'^[,\s]*(?:typename|class)(?:\.\.\.)?\s+(\w+)(.*)$', + argument_list) + if not match: + break + typenames.add(match.group(1)) + argument_list = match.group(2) + return typenames def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error): - """Check for rvalue references. + """Check for rvalue references. Args: filename: The name of the current file. @@ -3793,33 +3813,34 @@ def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error): the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # Find lines missing spaces around &&. - # TODO(unknown): currently we don't check for rvalue references - # with spaces surrounding the && to avoid false positives with - # boolean expressions. - line = clean_lines.elided[linenum] - match = Match(r'^(.*\S)&&', line) - if not match: - match = Match(r'(.*)&&\S', line) - if (not match) or '(&&)' in line or Search(r'\boperator\s*$', match.group(1)): - return - - # Either poorly formed && or an rvalue reference, check the context - # to get a more accurate error message. Mostly we want to determine - # if what's to the left of "&&" is a type or not. - typenames = GetTemplateArgs(clean_lines, linenum) - and_pos = len(match.group(1)) - if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos): - if not IsRValueAllowed(clean_lines, linenum, typenames): - error(filename, linenum, 'build/c++11', 3, - 'RValue references are an unapproved C++ feature.') - else: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around &&') + # Find lines missing spaces around &&. + # TODO(unknown): currently we don't check for rvalue references + # with spaces surrounding the && to avoid false positives with + # boolean expressions. + line = clean_lines.elided[linenum] + match = Match(r'^(.*\S)&&', line) + if not match: + match = Match(r'(.*)&&\S', line) + if (not match) or '(&&)' in line or Search(r'\boperator\s*$', + match.group(1)): + return + + # Either poorly formed && or an rvalue reference, check the context + # to get a more accurate error message. Mostly we want to determine + # if what's to the left of "&&" is a type or not. + typenames = GetTemplateArgs(clean_lines, linenum) + and_pos = len(match.group(1)) + if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos): + if not IsRValueAllowed(clean_lines, linenum, typenames): + error(filename, linenum, 'build/c++11', 3, + 'RValue references are an unapproved C++ feature.') + else: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around &&') def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): - """Checks for additional blank line issues related to sections. + """Checks for additional blank line issues related to sections. Currently the only thing checked here is blank line before protected/private. @@ -3830,51 +3851,53 @@ def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Skip checks if the class is small, where small means 25 lines or less. - # 25 lines seems like a good cutoff since that's the usual height of - # terminals, and any class that can't fit in one screen can't really - # be considered "small". - # - # Also skip checks if we are on the first line. This accounts for - # classes that look like - # class Foo { public: ... }; - # - # If we didn't find the end of the class, last_line would be zero, - # and the check will be skipped by the first condition. - if (class_info.last_line - class_info.starting_linenum <= 24 or - linenum <= class_info.starting_linenum): - return - - matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) - if matched: - # Issue warning if the line before public/protected/private was - # not a blank line, but don't do this if the previous line contains - # "class" or "struct". This can happen two ways: - # - We are at the beginning of the class. - # - We are forward-declaring an inner class that is semantically - # private, but needed to be public for implementation reasons. - # Also ignores cases where the previous line ends with a backslash as can be - # common when defining classes in C macros. - prev_line = clean_lines.lines[linenum - 1] - if (not IsBlankLine(prev_line) and - not Search(r'\b(class|struct)\b', prev_line) and - not Search(r'\\$', prev_line)): - # Try a bit harder to find the beginning of the class. This is to - # account for multi-line base-specifier lists, e.g.: - # class Derived - # : public Base { - end_class_head = class_info.starting_linenum - for i in range(class_info.starting_linenum, linenum): - if Search(r'\{\s*$', clean_lines.lines[i]): - end_class_head = i - break - if end_class_head < linenum - 1: - error(filename, linenum, 'whitespace/blank_line', 3, - '"%s:" should be preceded by a blank line' % matched.group(1)) + # Skip checks if the class is small, where small means 25 lines or less. + # 25 lines seems like a good cutoff since that's the usual height of + # terminals, and any class that can't fit in one screen can't really + # be considered "small". + # + # Also skip checks if we are on the first line. This accounts for + # classes that look like + # class Foo { public: ... }; + # + # If we didn't find the end of the class, last_line would be zero, + # and the check will be skipped by the first condition. + if (class_info.last_line - class_info.starting_linenum <= 24 or + linenum <= class_info.starting_linenum): + return + + matched = Match(r'\s*(public|protected|private):', + clean_lines.lines[linenum]) + if matched: + # Issue warning if the line before public/protected/private was + # not a blank line, but don't do this if the previous line contains + # "class" or "struct". This can happen two ways: + # - We are at the beginning of the class. + # - We are forward-declaring an inner class that is semantically + # private, but needed to be public for implementation reasons. + # Also ignores cases where the previous line ends with a backslash as can be + # common when defining classes in C macros. + prev_line = clean_lines.lines[linenum - 1] + if (not IsBlankLine(prev_line) and + not Search(r'\b(class|struct)\b', prev_line) and + not Search(r'\\$', prev_line)): + # Try a bit harder to find the beginning of the class. This is to + # account for multi-line base-specifier lists, e.g.: + # class Derived + # : public Base { + end_class_head = class_info.starting_linenum + for i in range(class_info.starting_linenum, linenum): + if Search(r'\{\s*$', clean_lines.lines[i]): + end_class_head = i + break + if end_class_head < linenum - 1: + error(filename, linenum, 'whitespace/blank_line', 3, + '"%s:" should be preceded by a blank line' % + matched.group(1)) def GetPreviousNonBlankLine(clean_lines, linenum): - """Return the most recent non-blank line and its line number. + """Return the most recent non-blank line and its line number. Args: clean_lines: A CleansedLines instance containing the file contents. @@ -3887,17 +3910,17 @@ def GetPreviousNonBlankLine(clean_lines, linenum): if this is the first non-blank line. """ - prevlinenum = linenum - 1 - while prevlinenum >= 0: - prevline = clean_lines.elided[prevlinenum] - if not IsBlankLine(prevline): # if not a blank line... - return (prevline, prevlinenum) - prevlinenum -= 1 - return ('', -1) + prevlinenum = linenum - 1 + while prevlinenum >= 0: + prevline = clean_lines.elided[prevlinenum] + if not IsBlankLine(prevline): # if not a blank line... + return (prevline, prevlinenum) + prevlinenum -= 1 + return ('', -1) def CheckBraces(filename, clean_lines, linenum, error): - """Looks for misplaced braces (e.g. at the end of line). + """Looks for misplaced braces (e.g. at the end of line). Args: filename: The name of the current file. @@ -3906,114 +3929,123 @@ def CheckBraces(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] # get rid of comments and strings - - if Match(r'\s*{\s*$', line): - # We allow an open brace to start a line in the case where someone is using - # braces in a block to explicitly create a new scope, which is commonly used - # to control the lifetime of stack-allocated variables. Braces are also - # used for brace initializers inside function calls. We don't detect this - # perfectly: we just don't complain if the last non-whitespace character on - # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the - # previous line starts a preprocessor block. - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if (not Search(r'[,;:}{(]\s*$', prevline) and - not Match(r'\s*#', prevline)): - error(filename, linenum, 'whitespace/braces', 4, - '{ should almost always be at the end of the previous line') - - # An else clause should be on the same line as the preceding closing brace. - if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if Match(r'\s*}\s*$', prevline): - error(filename, linenum, 'whitespace/newline', 4, - 'An else should appear on the same line as the preceding }') - - # If braces come on one side of an else, they should be on both. - # However, we have to worry about "else if" that spans multiple lines! - if Search(r'else if\s*\(', line): # could be multi-line if - brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) - # find the ( after the if - pos = line.find('else if') - pos = line.find('(', pos) - if pos > 0: - (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) - brace_on_right = endline[endpos:].find('{') != -1 - if brace_on_left != brace_on_right: # must be brace after if + line = clean_lines.elided[linenum] # get rid of comments and strings + + if Match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone is using + # braces in a block to explicitly create a new scope, which is commonly used + # to control the lifetime of stack-allocated variables. Braces are also + # used for brace initializers inside function calls. We don't detect this + # perfectly: we just don't complain if the last non-whitespace character on + # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the + # previous line starts a preprocessor block. + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if (not Search(r'[,;:}{(]\s*$', prevline) and + not Match(r'\s*#', prevline)): + error(filename, linenum, 'whitespace/braces', 4, + '{ should almost always be at the end of the previous line') + + # An else clause should be on the same line as the preceding closing brace. + if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if Match(r'\s*}\s*$', prevline): + error(filename, linenum, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') + + # If braces come on one side of an else, they should be on both. + # However, we have to worry about "else if" that spans multiple lines! + if Search(r'else if\s*\(', line): # could be multi-line if + brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) + # find the ( after the if + pos = line.find('else if') + pos = line.find('(', pos) + if pos > 0: + (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) + brace_on_right = endline[endpos:].find('{') != -1 + if brace_on_left != brace_on_right: # must be brace after if + error( + filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both' + ) + elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): error(filename, linenum, 'readability/braces', 5, 'If an else has a brace on one side, it should have it on both') - elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): - error(filename, linenum, 'readability/braces', 5, - 'If an else has a brace on one side, it should have it on both') - - # Likewise, an else should never have the else clause on the same line - if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): - error(filename, linenum, 'whitespace/newline', 4, - 'Else clause should never be on same line as else (use 2 lines)') - - # In the same way, a do/while should never be on one line - if Match(r'\s*do [^\s{]', line): - error(filename, linenum, 'whitespace/newline', 4, - 'do/while clauses should not be on a single line') - - # Check single-line if/else bodies. The style guide says 'curly braces are not - # required for single-line statements'. We additionally allow multi-line, - # single statements, but we reject anything with more than one semicolon in - # it. This means that the first semicolon after the if should be at the end of - # its line, and the line after that should have an indent level equal to or - # lower than the if. We also check for ambiguous if/else nesting without - # braces. - if_else_match = Search(r'\b(if\s*\(|else\b)', line) - if if_else_match and not Match(r'\s*#', line): - if_indent = GetIndentLevel(line) - endline, endlinenum, endpos = line, linenum, if_else_match.end() - if_match = Search(r'\bif\s*\(', line) - if if_match: - # This could be a multiline if condition, so find the end first. - pos = if_match.end() - 1 - (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos) - # Check for an opening brace, either directly after the if or on the next - # line. If found, this isn't a single-statement conditional. - if (not Match(r'\s*{', endline[endpos:]) - and not (Match(r'\s*$', endline[endpos:]) - and endlinenum < (len(clean_lines.elided) - 1) - and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): - while (endlinenum < len(clean_lines.elided) - and ';' not in clean_lines.elided[endlinenum][endpos:]): - endlinenum += 1 - endpos = 0 - if endlinenum < len(clean_lines.elided): - endline = clean_lines.elided[endlinenum] - # We allow a mix of whitespace and closing braces (e.g. for one-liner - # methods) and a single \ after the semicolon (for macros) - endpos = endline.find(';') - if not Match(r';[\s}]*(\\?)$', endline[endpos:]): - # Semicolon isn't the last character, there's something trailing. - # Output a warning if the semicolon is not contained inside - # a lambda expression. - if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', - endline): - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') - elif endlinenum < len(clean_lines.elided) - 1: - # Make sure the next line is dedented - next_line = clean_lines.elided[endlinenum + 1] - next_indent = GetIndentLevel(next_line) - # With ambiguous nested if statements, this will error out on the - # if that *doesn't* match the else, regardless of whether it's the - # inner one or outer one. - if (if_match and Match(r'\s*else\b', next_line) - and next_indent != if_indent): - error(filename, linenum, 'readability/braces', 4, - 'Else clause should be indented at the same level as if. ' - 'Ambiguous nested if/else chains require braces.') - elif next_indent > if_indent: - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') + + # Likewise, an else should never have the else clause on the same line + if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): + error(filename, linenum, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') + + # In the same way, a do/while should never be on one line + if Match(r'\s*do [^\s{]', line): + error(filename, linenum, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') + + # Check single-line if/else bodies. The style guide says 'curly braces are not + # required for single-line statements'. We additionally allow multi-line, + # single statements, but we reject anything with more than one semicolon in + # it. This means that the first semicolon after the if should be at the end of + # its line, and the line after that should have an indent level equal to or + # lower than the if. We also check for ambiguous if/else nesting without + # braces. + if_else_match = Search(r'\b(if\s*\(|else\b)', line) + if if_else_match and not Match(r'\s*#', line): + if_indent = GetIndentLevel(line) + endline, endlinenum, endpos = line, linenum, if_else_match.end() + if_match = Search(r'\bif\s*\(', line) + if if_match: + # This could be a multiline if condition, so find the end first. + pos = if_match.end() - 1 + (endline, endlinenum, endpos) = CloseExpression(clean_lines, + linenum, pos) + # Check for an opening brace, either directly after the if or on the next + # line. If found, this isn't a single-statement conditional. + if (not Match(r'\s*{', endline[endpos:]) and + not (Match(r'\s*$', endline[endpos:]) and endlinenum < + (len(clean_lines.elided) - 1) and + Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): + while (endlinenum < len(clean_lines.elided) and + ';' not in clean_lines.elided[endlinenum][endpos:]): + endlinenum += 1 + endpos = 0 + if endlinenum < len(clean_lines.elided): + endline = clean_lines.elided[endlinenum] + # We allow a mix of whitespace and closing braces (e.g. for one-liner + # methods) and a single \ after the semicolon (for macros) + endpos = endline.find(';') + if not Match(r';[\s}]*(\\?)$', endline[endpos:]): + # Semicolon isn't the last character, there's something trailing. + # Output a warning if the semicolon is not contained inside + # a lambda expression. + if not Match( + r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', + endline): + error( + filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces' + ) + elif endlinenum < len(clean_lines.elided) - 1: + # Make sure the next line is dedented + next_line = clean_lines.elided[endlinenum + 1] + next_indent = GetIndentLevel(next_line) + # With ambiguous nested if statements, this will error out on the + # if that *doesn't* match the else, regardless of whether it's the + # inner one or outer one. + if (if_match and Match(r'\s*else\b', next_line) and + next_indent != if_indent): + error( + filename, linenum, 'readability/braces', 4, + 'Else clause should be indented at the same level as if. ' + 'Ambiguous nested if/else chains require braces.') + elif next_indent > if_indent: + error( + filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces' + ) def CheckTrailingSemicolon(filename, clean_lines, linenum, error): - """Looks for redundant trailing semicolon. + """Looks for redundant trailing semicolon. Args: filename: The name of the current file. @@ -4022,135 +4054,133 @@ def CheckTrailingSemicolon(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Block bodies should not be followed by a semicolon. Due to C++11 - # brace initialization, there are more places where semicolons are - # required than not, so we use a whitelist approach to check these - # rather than a blacklist. These are the places where "};" should - # be replaced by just "}": - # 1. Some flavor of block following closing parenthesis: - # for (;;) {}; - # while (...) {}; - # switch (...) {}; - # Function(...) {}; - # if (...) {}; - # if (...) else if (...) {}; - # - # 2. else block: - # if (...) else {}; - # - # 3. const member function: - # Function(...) const {}; - # - # 4. Block following some statement: - # x = 42; - # {}; - # - # 5. Block at the beginning of a function: - # Function(...) { - # {}; - # } - # - # Note that naively checking for the preceding "{" will also match - # braces inside multi-dimensional arrays, but this is fine since - # that expression will not contain semicolons. - # - # 6. Block following another block: - # while (true) {} - # {}; - # - # 7. End of namespaces: - # namespace {}; - # - # These semicolons seems far more common than other kinds of - # redundant semicolons, possibly due to people converting classes - # to namespaces. For now we do not warn for this case. - # - # Try matching case 1 first. - match = Match(r'^(.*\)\s*)\{', line) - if match: - # Matched closing parenthesis (case 1). Check the token before the - # matching opening parenthesis, and don't warn if it looks like a - # macro. This avoids these false positives: - # - macro that defines a base class - # - multi-line macro that defines a base class - # - macro that defines the whole class-head + line = clean_lines.elided[linenum] + + # Block bodies should not be followed by a semicolon. Due to C++11 + # brace initialization, there are more places where semicolons are + # required than not, so we use a whitelist approach to check these + # rather than a blacklist. These are the places where "};" should + # be replaced by just "}": + # 1. Some flavor of block following closing parenthesis: + # for (;;) {}; + # while (...) {}; + # switch (...) {}; + # Function(...) {}; + # if (...) {}; + # if (...) else if (...) {}; # - # But we still issue warnings for macros that we know are safe to - # warn, specifically: - # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P - # - TYPED_TEST - # - INTERFACE_DEF - # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # 2. else block: + # if (...) else {}; # - # We implement a whitelist of safe macros instead of a blacklist of - # unsafe macros, even though the latter appears less frequently in - # google code and would have been easier to implement. This is because - # the downside for getting the whitelist wrong means some extra - # semicolons, while the downside for getting the blacklist wrong - # would result in compile errors. + # 3. const member function: + # Function(...) const {}; # - # In addition to macros, we also don't want to warn on - # - Compound literals - # - Lambdas - # - alignas specifier with anonymous structs: - closing_brace_pos = match.group(1).rfind(')') - opening_parenthesis = ReverseCloseExpression( - clean_lines, linenum, closing_brace_pos) - if opening_parenthesis[2] > -1: - line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] - macro = Search(r'\b([A-Z_]+)\s*$', line_prefix) - func = Match(r'^(.*\])\s*$', line_prefix) - if ((macro and - macro.group(1) not in ( - 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', - 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', - 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or - (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or - Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or - Search(r'\s+=\s*$', line_prefix)): - match = None - if (match and - opening_parenthesis[1] > 1 and - Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): - # Multi-line lambda-expression - match = None - - else: - # Try matching cases 2-3. - match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) - if not match: - # Try matching cases 4-6. These are always matched on separate lines. - # - # Note that we can't simply concatenate the previous line to the - # current line and do a single match, otherwise we may output - # duplicate warnings for the blank line case: - # if (cond) { - # // blank line - # } - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if prevline and Search(r'[;{}]\s*$', prevline): - match = Match(r'^(\s*)\{', line) - - # Check matching closing brace - if match: - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if endpos > -1 and Match(r'^\s*;', endline[endpos:]): - # Current {} pair is eligible for semicolon check, and we have found - # the redundant semicolon, output warning here. - # - # Note: because we are scanning forward for opening braces, and - # outputting warnings for the matching closing brace, if there are - # nested blocks with trailing semicolons, we will get the error - # messages in reversed order. - error(filename, endlinenum, 'readability/braces', 4, - "You don't need a ; after a }") + # 4. Block following some statement: + # x = 42; + # {}; + # + # 5. Block at the beginning of a function: + # Function(...) { + # {}; + # } + # + # Note that naively checking for the preceding "{" will also match + # braces inside multi-dimensional arrays, but this is fine since + # that expression will not contain semicolons. + # + # 6. Block following another block: + # while (true) {} + # {}; + # + # 7. End of namespaces: + # namespace {}; + # + # These semicolons seems far more common than other kinds of + # redundant semicolons, possibly due to people converting classes + # to namespaces. For now we do not warn for this case. + # + # Try matching case 1 first. + match = Match(r'^(.*\)\s*)\{', line) + if match: + # Matched closing parenthesis (case 1). Check the token before the + # matching opening parenthesis, and don't warn if it looks like a + # macro. This avoids these false positives: + # - macro that defines a base class + # - multi-line macro that defines a base class + # - macro that defines the whole class-head + # + # But we still issue warnings for macros that we know are safe to + # warn, specifically: + # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P + # - TYPED_TEST + # - INTERFACE_DEF + # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # + # We implement a whitelist of safe macros instead of a blacklist of + # unsafe macros, even though the latter appears less frequently in + # google code and would have been easier to implement. This is because + # the downside for getting the whitelist wrong means some extra + # semicolons, while the downside for getting the blacklist wrong + # would result in compile errors. + # + # In addition to macros, we also don't want to warn on + # - Compound literals + # - Lambdas + # - alignas specifier with anonymous structs: + closing_brace_pos = match.group(1).rfind(')') + opening_parenthesis = ReverseCloseExpression(clean_lines, linenum, + closing_brace_pos) + if opening_parenthesis[2] > -1: + line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] + macro = Search(r'\b([A-Z_]+)\s*$', line_prefix) + func = Match(r'^(.*\])\s*$', line_prefix) + if ((macro and macro.group(1) not in + ('TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', + 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', + 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or + (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or + Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or + Search(r'\s+=\s*$', line_prefix)): + match = None + if (match and opening_parenthesis[1] > 1 and Search( + r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): + # Multi-line lambda-expression + match = None + + else: + # Try matching cases 2-3. + match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) + if not match: + # Try matching cases 4-6. These are always matched on separate lines. + # + # Note that we can't simply concatenate the previous line to the + # current line and do a single match, otherwise we may output + # duplicate warnings for the blank line case: + # if (cond) { + # // blank line + # } + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if prevline and Search(r'[;{}]\s*$', prevline): + match = Match(r'^(\s*)\{', line) + + # Check matching closing brace + if match: + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if endpos > -1 and Match(r'^\s*;', endline[endpos:]): + # Current {} pair is eligible for semicolon check, and we have found + # the redundant semicolon, output warning here. + # + # Note: because we are scanning forward for opening braces, and + # outputting warnings for the matching closing brace, if there are + # nested blocks with trailing semicolons, we will get the error + # messages in reversed order. + error(filename, endlinenum, 'readability/braces', 4, + "You don't need a ; after a }") def CheckEmptyBlockBody(filename, clean_lines, linenum, error): - """Look for empty loop/conditional body with only a single semicolon. + """Look for empty loop/conditional body with only a single semicolon. Args: filename: The name of the current file. @@ -4159,33 +4189,34 @@ def CheckEmptyBlockBody(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - # Search for loop keywords at the beginning of the line. Because only - # whitespaces are allowed before the keywords, this will also ignore most - # do-while-loops, since those lines should start with closing brace. - # - # We also check "if" blocks here, since an empty conditional block - # is likely an error. - line = clean_lines.elided[linenum] - matched = Match(r'\s*(for|while|if)\s*\(', line) - if matched: - # Find the end of the conditional expression - (end_line, end_linenum, end_pos) = CloseExpression( - clean_lines, linenum, line.find('(')) - - # Output warning if what follows the condition expression is a semicolon. - # No warning for all other cases, including whitespace or newline, since we - # have a separate check for semicolons preceded by whitespace. - if end_pos >= 0 and Match(r';', end_line[end_pos:]): - if matched.group(1) == 'if': - error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, - 'Empty conditional bodies should use {}') - else: - error(filename, end_linenum, 'whitespace/empty_loop_body', 5, - 'Empty loop bodies should use {} or continue') + # Search for loop keywords at the beginning of the line. Because only + # whitespaces are allowed before the keywords, this will also ignore most + # do-while-loops, since those lines should start with closing brace. + # + # We also check "if" blocks here, since an empty conditional block + # is likely an error. + line = clean_lines.elided[linenum] + matched = Match(r'\s*(for|while|if)\s*\(', line) + if matched: + # Find the end of the conditional expression + (end_line, end_linenum, end_pos) = CloseExpression(clean_lines, linenum, + line.find('(')) + + # Output warning if what follows the condition expression is a semicolon. + # No warning for all other cases, including whitespace or newline, since we + # have a separate check for semicolons preceded by whitespace. + if end_pos >= 0 and Match(r';', end_line[end_pos:]): + if matched.group(1) == 'if': + error(filename, end_linenum, + 'whitespace/empty_conditional_body', 5, + 'Empty conditional bodies should use {}') + else: + error(filename, end_linenum, 'whitespace/empty_loop_body', 5, + 'Empty loop bodies should use {} or continue') def FindCheckMacro(line): - """Find a replaceable CHECK-like macro. + """Find a replaceable CHECK-like macro. Args: line: line to search on. @@ -4193,22 +4224,22 @@ def FindCheckMacro(line): (macro name, start position), or (None, -1) if no replaceable macro is found. """ - for macro in _CHECK_MACROS: - i = line.find(macro) - if i >= 0: - # Find opening parenthesis. Do a regular expression match here - # to make sure that we are matching the expected CHECK macro, as - # opposed to some other macro that happens to contain the CHECK - # substring. - matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) - if not matched: - continue - return (macro, len(matched.group(1))) - return (None, -1) + for macro in _CHECK_MACROS: + i = line.find(macro) + if i >= 0: + # Find opening parenthesis. Do a regular expression match here + # to make sure that we are matching the expected CHECK macro, as + # opposed to some other macro that happens to contain the CHECK + # substring. + matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) + if not matched: + continue + return (macro, len(matched.group(1))) + return (None, -1) def CheckCheck(filename, clean_lines, linenum, error): - """Checks the use of CHECK and EXPECT macros. + """Checks the use of CHECK and EXPECT macros. Args: filename: The name of the current file. @@ -4217,116 +4248,116 @@ def CheckCheck(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - # Decide the set of replacement macros that should be suggested - lines = clean_lines.elided - (check_macro, start_pos) = FindCheckMacro(lines[linenum]) - if not check_macro: - return - - # Find end of the boolean expression by matching parentheses - (last_line, end_line, end_pos) = CloseExpression( - clean_lines, linenum, start_pos) - if end_pos < 0: - return - - # If the check macro is followed by something other than a - # semicolon, assume users will log their own custom error messages - # and don't suggest any replacements. - if not Match(r'\s*;', last_line[end_pos:]): - return - - if linenum == end_line: - expression = lines[linenum][start_pos + 1:end_pos - 1] - else: - expression = lines[linenum][start_pos + 1:] - for i in xrange(linenum + 1, end_line): - expression += lines[i] - expression += last_line[0:end_pos - 1] - - # Parse expression so that we can take parentheses into account. - # This avoids false positives for inputs like "CHECK((a < 4) == b)", - # which is not replaceable by CHECK_LE. - lhs = '' - rhs = '' - operator = None - while expression: - matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' - r'==|!=|>=|>|<=|<|\()(.*)$', expression) - if matched: - token = matched.group(1) - if token == '(': - # Parenthesized operand - expression = matched.group(2) - (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) - if end < 0: - return # Unmatched parenthesis - lhs += '(' + expression[0:end] - expression = expression[end:] - elif token in ('&&', '||'): - # Logical and/or operators. This means the expression - # contains more than one term, for example: - # CHECK(42 < a && a < b); - # - # These are not replaceable with CHECK_LE, so bail out early. + # Decide the set of replacement macros that should be suggested + lines = clean_lines.elided + (check_macro, start_pos) = FindCheckMacro(lines[linenum]) + if not check_macro: return - elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): - # Non-relational operator - lhs += token - expression = matched.group(2) - else: - # Relational operator - operator = token - rhs = matched.group(2) - break + + # Find end of the boolean expression by matching parentheses + (last_line, end_line, end_pos) = CloseExpression(clean_lines, linenum, + start_pos) + if end_pos < 0: + return + + # If the check macro is followed by something other than a + # semicolon, assume users will log their own custom error messages + # and don't suggest any replacements. + if not Match(r'\s*;', last_line[end_pos:]): + return + + if linenum == end_line: + expression = lines[linenum][start_pos + 1:end_pos - 1] else: - # Unparenthesized operand. Instead of appending to lhs one character - # at a time, we do another regular expression match to consume several - # characters at once if possible. Trivial benchmark shows that this - # is more efficient when the operands are longer than a single - # character, which is generally the case. - matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) - if not matched: - matched = Match(r'^(\s*\S)(.*)$', expression) - if not matched: - break - lhs += matched.group(1) - expression = matched.group(2) - - # Only apply checks if we got all parts of the boolean expression - if not (lhs and operator and rhs): - return - - # Check that rhs do not contain logical operators. We already know - # that lhs is fine since the loop above parses out && and ||. - if rhs.find('&&') > -1 or rhs.find('||') > -1: - return - - # At least one of the operands must be a constant literal. This is - # to avoid suggesting replacements for unprintable things like - # CHECK(variable != iterator) - # - # The following pattern matches decimal, hex integers, strings, and - # characters (in that order). - lhs = lhs.strip() - rhs = rhs.strip() - match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' - if Match(match_constant, lhs) or Match(match_constant, rhs): - # Note: since we know both lhs and rhs, we can provide a more - # descriptive error message like: - # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) - # Instead of: - # Consider using CHECK_EQ instead of CHECK(a == b) + expression = lines[linenum][start_pos + 1:] + for i in xrange(linenum + 1, end_line): + expression += lines[i] + expression += last_line[0:end_pos - 1] + + # Parse expression so that we can take parentheses into account. + # This avoids false positives for inputs like "CHECK((a < 4) == b)", + # which is not replaceable by CHECK_LE. + lhs = '' + rhs = '' + operator = None + while expression: + matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' + r'==|!=|>=|>|<=|<|\()(.*)$', expression) + if matched: + token = matched.group(1) + if token == '(': + # Parenthesized operand + expression = matched.group(2) + (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) + if end < 0: + return # Unmatched parenthesis + lhs += '(' + expression[0:end] + expression = expression[end:] + elif token in ('&&', '||'): + # Logical and/or operators. This means the expression + # contains more than one term, for example: + # CHECK(42 < a && a < b); + # + # These are not replaceable with CHECK_LE, so bail out early. + return + elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): + # Non-relational operator + lhs += token + expression = matched.group(2) + else: + # Relational operator + operator = token + rhs = matched.group(2) + break + else: + # Unparenthesized operand. Instead of appending to lhs one character + # at a time, we do another regular expression match to consume several + # characters at once if possible. Trivial benchmark shows that this + # is more efficient when the operands are longer than a single + # character, which is generally the case. + matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) + if not matched: + matched = Match(r'^(\s*\S)(.*)$', expression) + if not matched: + break + lhs += matched.group(1) + expression = matched.group(2) + + # Only apply checks if we got all parts of the boolean expression + if not (lhs and operator and rhs): + return + + # Check that rhs do not contain logical operators. We already know + # that lhs is fine since the loop above parses out && and ||. + if rhs.find('&&') > -1 or rhs.find('||') > -1: + return + + # At least one of the operands must be a constant literal. This is + # to avoid suggesting replacements for unprintable things like + # CHECK(variable != iterator) # - # We are still keeping the less descriptive message because if lhs - # or rhs gets long, the error message might become unreadable. - error(filename, linenum, 'readability/check', 2, - 'Consider using %s instead of %s(a %s b)' % ( - _CHECK_REPLACEMENT[check_macro][operator], - check_macro, operator)) + # The following pattern matches decimal, hex integers, strings, and + # characters (in that order). + lhs = lhs.strip() + rhs = rhs.strip() + match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' + if Match(match_constant, lhs) or Match(match_constant, rhs): + # Note: since we know both lhs and rhs, we can provide a more + # descriptive error message like: + # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) + # Instead of: + # Consider using CHECK_EQ instead of CHECK(a == b) + # + # We are still keeping the less descriptive message because if lhs + # or rhs gets long, the error message might become unreadable. + error(filename, linenum, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % + (_CHECK_REPLACEMENT[check_macro][operator], check_macro, + operator)) def CheckAltTokens(filename, clean_lines, linenum, error): - """Check alternative keywords being used in boolean expressions. + """Check alternative keywords being used in boolean expressions. Args: filename: The name of the current file. @@ -4334,31 +4365,31 @@ def CheckAltTokens(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Avoid preprocessor lines - if Match(r'^\s*#', line): - return + # Avoid preprocessor lines + if Match(r'^\s*#', line): + return - # Last ditch effort to avoid multi-line comments. This will not help - # if the comment started before the current line or ended after the - # current line, but it catches most of the false positives. At least, - # it provides a way to workaround this warning for people who use - # multi-line comments in preprocessor macros. - # - # TODO(unknown): remove this once cpplint has better support for - # multi-line comments. - if line.find('/*') >= 0 or line.find('*/') >= 0: - return + # Last ditch effort to avoid multi-line comments. This will not help + # if the comment started before the current line or ended after the + # current line, but it catches most of the false positives. At least, + # it provides a way to workaround this warning for people who use + # multi-line comments in preprocessor macros. + # + # TODO(unknown): remove this once cpplint has better support for + # multi-line comments. + if line.find('/*') >= 0 or line.find('*/') >= 0: + return - for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): - error(filename, linenum, 'readability/alt_tokens', 2, - 'Use operator %s instead of %s' % ( - _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) + for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): + error(filename, linenum, 'readability/alt_tokens', 2, + 'Use operator %s instead of %s' % ( + _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) def GetLineWidth(line): - """Determines the width of the line in column positions. + """Determines the width of the line in column positions. Args: line: A string, which may be a Unicode string. @@ -4367,21 +4398,21 @@ def GetLineWidth(line): The width of the line in column positions, accounting for Unicode combining characters and wide characters. """ - if isinstance(line, unicode): - width = 0 - for uc in unicodedata.normalize('NFC', line): - if unicodedata.east_asian_width(uc) in ('W', 'F'): - width += 2 - elif not unicodedata.combining(uc): - width += 1 - return width - else: - return len(line) + if isinstance(line, unicode): + width = 0 + for uc in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(uc) in ('W', 'F'): + width += 2 + elif not unicodedata.combining(uc): + width += 1 + return width + else: + return len(line) def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, error): - """Checks rules from the 'C++ style rules' section of cppguide.html. + """Checks rules from the 'C++ style rules' section of cppguide.html. Most of these rules are hard to test (naming, comment style), but we do what we can. In particular we check for 2-space indents, line lengths, @@ -4397,105 +4428,105 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, error: The function to call with any errors found. """ - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw_lines = clean_lines.lines_without_raw_strings - line = raw_lines[linenum] - - if line.find('\t') != -1: - error(filename, linenum, 'whitespace/tab', 1, - 'Tab found; better to use spaces') - - # One or three blank spaces at the beginning of the line is weird; it's - # hard to reconcile that with 2-space indents. - # NOTE: here are the conditions rob pike used for his tests. Mine aren't - # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces - # if(RLENGTH > 20) complain = 0; - # if(match($0, " +(error|private|public|protected):")) complain = 0; - # if(match(prev, "&& *$")) complain = 0; - # if(match(prev, "\\|\\| *$")) complain = 0; - # if(match(prev, "[\",=><] *$")) complain = 0; - # if(match($0, " <<")) complain = 0; - # if(match(prev, " +for \\(")) complain = 0; - # if(prevodd && match(prevprev, " +for \\(")) complain = 0; - scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' - classinfo = nesting_state.InnermostClass() - initial_spaces = 0 - cleansed_line = clean_lines.elided[linenum] - while initial_spaces < len(line) and line[initial_spaces] == ' ': - initial_spaces += 1 - if line and line[-1].isspace(): - error(filename, linenum, 'whitespace/end_of_line', 4, - 'Line ends in whitespace. Consider deleting these extra spaces.') - # There are certain situations we allow one space, notably for - # section labels, and also lines containing multi-line raw strings. - elif ((initial_spaces == 1 or initial_spaces == 3) and - not Match(scope_or_label_pattern, cleansed_line) and - not (clean_lines.raw_lines[linenum] != line and - Match(r'^\s*""', line))): - error(filename, linenum, 'whitespace/indent', 3, - 'Weird number of spaces at line-start. ' - 'Are you using a 2-space indent?') - - # Check if the line is a header guard. - is_header_guard = False - if file_extension == 'h': - cppvar = GetHeaderGuardCPPVariable(filename) - if (line.startswith('#ifndef %s' % cppvar) or - line.startswith('#define %s' % cppvar) or - line.startswith('#endif // %s' % cppvar)): - is_header_guard = True - # #include lines and header guards can be long, since there's no clean way to - # split them. - # - # URLs can be long too. It's possible to split these, but it makes them - # harder to cut&paste. - # - # The "$Id:...$" comment may also get very long without it being the - # developers fault. - if (not line.startswith('#include') and not is_header_guard and - not Match(r'^\s*//.*http(s?)://\S*$', line) and - not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): - line_width = GetLineWidth(line) - extended_length = int((_line_length * 1.25)) - if line_width > extended_length: - error(filename, linenum, 'whitespace/line_length', 4, - 'Lines should very rarely be longer than %i characters' % - extended_length) - elif line_width > _line_length: - error(filename, linenum, 'whitespace/line_length', 2, - 'Lines should be <= %i characters long' % _line_length) - - if (cleansed_line.count(';') > 1 and - # for loops are allowed two ;'s (and may run over two lines). - cleansed_line.find('for') == -1 and - (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or - GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and - # It's ok to have many commands in a switch case that fits in 1 line - not ((cleansed_line.find('case ') != -1 or - cleansed_line.find('default:') != -1) and - cleansed_line.find('break;') != -1)): - error(filename, linenum, 'whitespace/newline', 0, - 'More than one command on the same line') - - # Some more style checks - CheckBraces(filename, clean_lines, linenum, error) - CheckTrailingSemicolon(filename, clean_lines, linenum, error) - CheckEmptyBlockBody(filename, clean_lines, linenum, error) - CheckAccess(filename, clean_lines, linenum, nesting_state, error) - CheckSpacing(filename, clean_lines, linenum, nesting_state, error) - CheckOperatorSpacing(filename, clean_lines, linenum, error) - CheckParenthesisSpacing(filename, clean_lines, linenum, error) - CheckCommaSpacing(filename, clean_lines, linenum, error) - CheckBracesSpacing(filename, clean_lines, linenum, error) - CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) - CheckRValueReference(filename, clean_lines, linenum, nesting_state, error) - CheckCheck(filename, clean_lines, linenum, error) - CheckAltTokens(filename, clean_lines, linenum, error) - classinfo = nesting_state.InnermostClass() - if classinfo: - CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw_lines = clean_lines.lines_without_raw_strings + line = raw_lines[linenum] + + if line.find('\t') != -1: + error(filename, linenum, 'whitespace/tab', 1, + 'Tab found; better to use spaces') + + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 2-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces + # if(RLENGTH > 20) complain = 0; + # if(match($0, " +(error|private|public|protected):")) complain = 0; + # if(match(prev, "&& *$")) complain = 0; + # if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; + # if(match($0, " <<")) complain = 0; + # if(match(prev, " +for \\(")) complain = 0; + # if(prevodd && match(prevprev, " +for \\(")) complain = 0; + scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' + classinfo = nesting_state.InnermostClass() + initial_spaces = 0 + cleansed_line = clean_lines.elided[linenum] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + if line and line[-1].isspace(): + error(filename, linenum, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') + # There are certain situations we allow one space, notably for + # section labels, and also lines containing multi-line raw strings. + elif ((initial_spaces == 1 or initial_spaces == 3) and + not Match(scope_or_label_pattern, cleansed_line) and + not (clean_lines.raw_lines[linenum] != line and + Match(r'^\s*""', line))): + error(filename, linenum, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using a 2-space indent?') + + # Check if the line is a header guard. + is_header_guard = False + if file_extension == 'h': + cppvar = GetHeaderGuardCPPVariable(filename) + if (line.startswith('#ifndef %s' % cppvar) or + line.startswith('#define %s' % cppvar) or + line.startswith('#endif // %s' % cppvar)): + is_header_guard = True + # #include lines and header guards can be long, since there's no clean way to + # split them. + # + # URLs can be long too. It's possible to split these, but it makes them + # harder to cut&paste. + # + # The "$Id:...$" comment may also get very long without it being the + # developers fault. + if (not line.startswith('#include') and not is_header_guard and + not Match(r'^\s*//.*http(s?)://\S*$', line) and + not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): + line_width = GetLineWidth(line) + extended_length = int((_line_length * 1.25)) + if line_width > extended_length: + error(filename, linenum, 'whitespace/line_length', 4, + 'Lines should very rarely be longer than %i characters' % + extended_length) + elif line_width > _line_length: + error(filename, linenum, 'whitespace/line_length', 2, + 'Lines should be <= %i characters long' % _line_length) + + if (cleansed_line.count(';') > 1 and + # for loops are allowed two ;'s (and may run over two lines). + cleansed_line.find('for') == -1 and + (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or + GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and + # It's ok to have many commands in a switch case that fits in 1 line + not ((cleansed_line.find('case ') != -1 or + cleansed_line.find('default:') != -1) and + cleansed_line.find('break;') != -1)): + error(filename, linenum, 'whitespace/newline', 0, + 'More than one command on the same line') + + # Some more style checks + CheckBraces(filename, clean_lines, linenum, error) + CheckTrailingSemicolon(filename, clean_lines, linenum, error) + CheckEmptyBlockBody(filename, clean_lines, linenum, error) + CheckAccess(filename, clean_lines, linenum, nesting_state, error) + CheckSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckOperatorSpacing(filename, clean_lines, linenum, error) + CheckParenthesisSpacing(filename, clean_lines, linenum, error) + CheckCommaSpacing(filename, clean_lines, linenum, error) + CheckBracesSpacing(filename, clean_lines, linenum, error) + CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) + CheckRValueReference(filename, clean_lines, linenum, nesting_state, error) + CheckCheck(filename, clean_lines, linenum, error) + CheckAltTokens(filename, clean_lines, linenum, error) + classinfo = nesting_state.InnermostClass() + if classinfo: + CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) _RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') @@ -4508,7 +4539,7 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, def _DropCommonSuffixes(filename): - """Drops common suffixes like _test.cc or -inl.h from filename. + """Drops common suffixes like _test.cc or -inl.h from filename. For example: >>> _DropCommonSuffixes('foo/foo-inl.h') @@ -4526,16 +4557,16 @@ def _DropCommonSuffixes(filename): Returns: The filename with the common suffix removed. """ - for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', - 'inl.h', 'impl.h', 'internal.h'): - if (filename.endswith(suffix) and len(filename) > len(suffix) and - filename[-len(suffix) - 1] in ('-', '_')): - return filename[:-len(suffix) - 1] - return os.path.splitext(filename)[0] + for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', 'inl.h', 'impl.h', + 'internal.h'): + if (filename.endswith(suffix) and len(filename) > len(suffix) and + filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] def _IsTestFilename(filename): - """Determines if the given filename has a suffix that identifies it as a test. + """Determines if the given filename has a suffix that identifies it as a test. Args: filename: The input filename. @@ -4543,16 +4574,15 @@ def _IsTestFilename(filename): Returns: True if 'filename' looks like a test, False otherwise. """ - if (filename.endswith('_test.cc') or - filename.endswith('_unittest.cc') or - filename.endswith('_regtest.cc')): - return True - else: - return False + if (filename.endswith('_test.cc') or filename.endswith('_unittest.cc') or + filename.endswith('_regtest.cc')): + return True + else: + return False def _ClassifyInclude(fileinfo, include, is_system): - """Figures out what kind of header 'include' is. + """Figures out what kind of header 'include' is. Args: fileinfo: The current file cpplint is running over. A FileInfo instance. @@ -4575,44 +4605,43 @@ def _ClassifyInclude(fileinfo, include, is_system): >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) _OTHER_HEADER """ - # This is a list of all standard c++ header files, except - # those already checked for above. - is_cpp_h = include in _CPP_HEADERS - - if is_system: - if is_cpp_h: - return _CPP_SYS_HEADER - else: - return _C_SYS_HEADER - - # If the target file and the include we're checking share a - # basename when we drop common extensions, and the include - # lives in . , then it's likely to be owned by the target file. - target_dir, target_base = ( - os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) - include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) - if target_base == include_base and ( - include_dir == target_dir or - include_dir == os.path.normpath(target_dir + '/../public')): - return _LIKELY_MY_HEADER - - # If the target and include share some initial basename - # component, it's possible the target is implementing the - # include, so it's allowed to be first, but we'll never - # complain if it's not there. - target_first_component = _RE_FIRST_COMPONENT.match(target_base) - include_first_component = _RE_FIRST_COMPONENT.match(include_base) - if (target_first_component and include_first_component and - target_first_component.group(0) == - include_first_component.group(0)): - return _POSSIBLE_MY_HEADER - - return _OTHER_HEADER + # This is a list of all standard c++ header files, except + # those already checked for above. + is_cpp_h = include in _CPP_HEADERS + if is_system: + if is_cpp_h: + return _CPP_SYS_HEADER + else: + return _C_SYS_HEADER + + # If the target file and the include we're checking share a + # basename when we drop common extensions, and the include + # lives in . , then it's likely to be owned by the target file. + target_dir, target_base = ( + os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) + include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) + if target_base == include_base and ( + include_dir == target_dir or + include_dir == os.path.normpath(target_dir + '/../public')): + return _LIKELY_MY_HEADER + + # If the target and include share some initial basename + # component, it's possible the target is implementing the + # include, so it's allowed to be first, but we'll never + # complain if it's not there. + target_first_component = _RE_FIRST_COMPONENT.match(target_base) + include_first_component = _RE_FIRST_COMPONENT.match(include_base) + if (target_first_component and include_first_component and + target_first_component.group(0) == + include_first_component.group(0)): + return _POSSIBLE_MY_HEADER + + return _OTHER_HEADER def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): - """Check rules that are applicable to #include lines. + """Check rules that are applicable to #include lines. Strings on #include lines are NOT removed from elided line, to make certain tasks easier. However, to prevent false positives, checks @@ -4625,68 +4654,69 @@ def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): include_state: An _IncludeState instance in which the headers are inserted. error: The function to call with any errors found. """ - fileinfo = FileInfo(filename) - line = clean_lines.lines[linenum] - - # "include" should use the new style "foo/bar.h" instead of just "bar.h" - # Only do this check if the included header follows google naming - # conventions. If not, assume that it's a 3rd party API that - # requires special include conventions. - # - # We also make an exception for Lua headers, which follow google - # naming convention but not the include convention. - match = Match(r'#include\s*"([^/]+\.h)"', line) - if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): - error(filename, linenum, 'build/include', 4, - 'Include the directory when naming .h files') - - # we shouldn't include a file more than once. actually, there are a - # handful of instances where doing so is okay, but in general it's - # not. - match = _RE_PATTERN_INCLUDE.search(line) - if match: - include = match.group(2) - is_system = (match.group(1) == '<') - duplicate_line = include_state.FindHeader(include) - if duplicate_line >= 0: - error(filename, linenum, 'build/include', 4, - '"%s" already included at %s:%s' % - (include, filename, duplicate_line)) - elif (include.endswith('.cc') and - os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)): - error(filename, linenum, 'build/include', 4, - 'Do not include .cc files from other packages') - elif not _THIRD_PARTY_HEADERS_PATTERN.match(include): - include_state.include_list[-1].append((include, linenum)) - - # We want to ensure that headers appear in the right order: - # 1) for foo.cc, foo.h (preferred location) - # 2) c system files - # 3) cpp system files - # 4) for foo.cc, foo.h (deprecated location) - # 5) other google headers - # - # We classify each include statement as one of those 5 types - # using a number of techniques. The include_state object keeps - # track of the highest type seen, and complains if we see a - # lower type after that. - error_message = include_state.CheckNextIncludeOrder( - _ClassifyInclude(fileinfo, include, is_system)) - if error_message: - error(filename, linenum, 'build/include_order', 4, - '%s. Should be: %s.h, c system, c++ system, other.' % - (error_message, fileinfo.BaseName())) - canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) - if not include_state.IsInAlphabeticalOrder( - clean_lines, linenum, canonical_include): - error(filename, linenum, 'build/include_alpha', 4, - 'Include "%s" not in alphabetical order' % include) - include_state.SetLastHeader(canonical_include) + fileinfo = FileInfo(filename) + line = clean_lines.lines[linenum] + # "include" should use the new style "foo/bar.h" instead of just "bar.h" + # Only do this check if the included header follows google naming + # conventions. If not, assume that it's a 3rd party API that + # requires special include conventions. + # + # We also make an exception for Lua headers, which follow google + # naming convention but not the include convention. + match = Match(r'#include\s*"([^/]+\.h)"', line) + if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): + error(filename, linenum, 'build/include', 4, + 'Include the directory when naming .h files') + + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + match = _RE_PATTERN_INCLUDE.search(line) + if match: + include = match.group(2) + is_system = (match.group(1) == '<') + duplicate_line = include_state.FindHeader(include) + if duplicate_line >= 0: + error(filename, linenum, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, duplicate_line)) + elif (include.endswith('.cc') and + os.path.dirname(fileinfo.RepositoryName()) != + os.path.dirname(include)): + error(filename, linenum, 'build/include', 4, + 'Do not include .cc files from other packages') + elif not _THIRD_PARTY_HEADERS_PATTERN.match(include): + include_state.include_list[-1].append((include, linenum)) + + # We want to ensure that headers appear in the right order: + # 1) for foo.cc, foo.h (preferred location) + # 2) c system files + # 3) cpp system files + # 4) for foo.cc, foo.h (deprecated location) + # 5) other google headers + # + # We classify each include statement as one of those 5 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + error_message = include_state.CheckNextIncludeOrder( + _ClassifyInclude(fileinfo, include, is_system)) + if error_message: + error(filename, linenum, 'build/include_order', 4, + '%s. Should be: %s.h, c system, c++ system, other.' % + (error_message, fileinfo.BaseName())) + canonical_include = include_state.CanonicalizeAlphabeticalOrder( + include) + if not include_state.IsInAlphabeticalOrder(clean_lines, linenum, + canonical_include): + error(filename, linenum, 'build/include_alpha', 4, + 'Include "%s" not in alphabetical order' % include) + include_state.SetLastHeader(canonical_include) def _GetTextInside(text, start_pattern): - r"""Retrieves all the text between matching open and close parentheses. + r"""Retrieves all the text between matching open and close parentheses. Given a string of lines and a regular expression string, retrieve all the text following the expression and between opening punctuation symbols like @@ -4705,40 +4735,40 @@ def _GetTextInside(text, start_pattern): The extracted text. None if either the opening string or ending punctuation could not be found. """ - # TODO(unknown): Audit cpplint.py to see what places could be profitably - # rewritten to use _GetTextInside (and use inferior regexp matching today). - - # Give opening punctuations to get the matching close-punctuations. - matching_punctuation = {'(': ')', '{': '}', '[': ']'} - closing_punctuation = set(matching_punctuation.itervalues()) - - # Find the position to start extracting text. - match = re.search(start_pattern, text, re.M) - if not match: # start_pattern not found in text. - return None - start_position = match.end(0) - - assert start_position > 0, ( - 'start_pattern must ends with an opening punctuation.') - assert text[start_position - 1] in matching_punctuation, ( - 'start_pattern must ends with an opening punctuation.') - # Stack of closing punctuations we expect to have in text after position. - punctuation_stack = [matching_punctuation[text[start_position - 1]]] - position = start_position - while punctuation_stack and position < len(text): - if text[position] == punctuation_stack[-1]: - punctuation_stack.pop() - elif text[position] in closing_punctuation: - # A closing punctuation without matching opening punctuations. - return None - elif text[position] in matching_punctuation: - punctuation_stack.append(matching_punctuation[text[position]]) - position += 1 - if punctuation_stack: - # Opening punctuations left without matching close-punctuations. - return None - # punctuations match. - return text[start_position:position - 1] + # TODO(unknown): Audit cpplint.py to see what places could be profitably + # rewritten to use _GetTextInside (and use inferior regexp matching today). + + # Give opening punctuations to get the matching close-punctuations. + matching_punctuation = {'(': ')', '{': '}', '[': ']'} + closing_punctuation = set(matching_punctuation.itervalues()) + + # Find the position to start extracting text. + match = re.search(start_pattern, text, re.M) + if not match: # start_pattern not found in text. + return None + start_position = match.end(0) + + assert start_position > 0, ( + 'start_pattern must ends with an opening punctuation.') + assert text[start_position - 1] in matching_punctuation, ( + 'start_pattern must ends with an opening punctuation.') + # Stack of closing punctuations we expect to have in text after position. + punctuation_stack = [matching_punctuation[text[start_position - 1]]] + position = start_position + while punctuation_stack and position < len(text): + if text[position] == punctuation_stack[-1]: + punctuation_stack.pop() + elif text[position] in closing_punctuation: + # A closing punctuation without matching opening punctuations. + return None + elif text[position] in matching_punctuation: + punctuation_stack.append(matching_punctuation[text[position]]) + position += 1 + if punctuation_stack: + # Opening punctuations left without matching close-punctuations. + return None + # punctuations match. + return text[start_position:position - 1] # Patterns for matching call-by-reference parameters. @@ -4763,13 +4793,13 @@ def _GetTextInside(text, start_pattern): # A call-by-const-reference parameter either ends with 'const& identifier' # or looks like 'const type& identifier' when 'type' is atomic. _RE_PATTERN_CONST_REF_PARAM = ( - r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + - r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') + r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + r'|const\s+' + + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') -def CheckLanguage(filename, clean_lines, linenum, file_extension, - include_state, nesting_state, error): - """Checks rules from the 'C++ language rules' section of cppguide.html. +def CheckLanguage(filename, clean_lines, linenum, file_extension, include_state, + nesting_state, error): + """Checks rules from the 'C++ language rules' section of cppguide.html. Some of these rules are hard to test (function overloading, using uint32 inappropriately), but we do the best we can. @@ -4784,149 +4814,152 @@ def CheckLanguage(filename, clean_lines, linenum, file_extension, the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # If the line is empty or consists of entirely a comment, no need to - # check it. - line = clean_lines.elided[linenum] - if not line: - return - - match = _RE_PATTERN_INCLUDE.search(line) - if match: - CheckIncludeLine(filename, clean_lines, linenum, include_state, error) - return - - # Reset include state across preprocessor directives. This is meant - # to silence warnings for conditional includes. - match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) - if match: - include_state.ResetSection(match.group(1)) - - # Make Windows paths like Unix. - fullname = os.path.abspath(filename).replace('\\', '/') - - # Perform other checks now that we are sure that this is not an include line - CheckCasts(filename, clean_lines, linenum, error) - CheckGlobalStatic(filename, clean_lines, linenum, error) - CheckPrintf(filename, clean_lines, linenum, error) - - if file_extension == 'h': - # TODO(unknown): check that 1-arg constructors are explicit. - # How to tell it's a constructor? - # (handled in CheckForNonStandardConstructs for now) - # TODO(unknown): check that classes declare or disable copy/assign - # (level 1 error) - pass + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[linenum] + if not line: + return - # Check if people are using the verboten C basic types. The only exception - # we regularly allow is "unsigned short port" for port. - if Search(r'\bshort port\b', line): - if not Search(r'\bunsigned short port\b', line): - error(filename, linenum, 'runtime/int', 4, - 'Use "unsigned short" for ports, not "short"') - else: - match = Search(r'\b(short|long(?! +double)|long long)\b', line) + match = _RE_PATTERN_INCLUDE.search(line) if match: - error(filename, linenum, 'runtime/int', 4, - 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) - - # Check if some verboten operator overloading is going on - # TODO(unknown): catch out-of-line unary operator&: - # class X {}; - # int operator&(const X& x) { return 42; } // unary operator& - # The trick is it's hard to tell apart from binary operator&: - # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& - if Search(r'\boperator\s*&\s*\(\s*\)', line): - error(filename, linenum, 'runtime/operator', 4, - 'Unary operator& is dangerous. Do not use it.') - - # Check for suspicious usage of "if" like - # } if (a == b) { - if Search(r'\}\s*if\s*\(', line): - error(filename, linenum, 'readability/braces', 4, - 'Did you mean "else if"? If not, start a new line for "if".') - - # Check for potential format string bugs like printf(foo). - # We constrain the pattern not to pick things like DocidForPrintf(foo). - # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) - # TODO(unknown): Catch the following case. Need to change the calling - # convention of the whole function to process multiple line to handle it. - # printf( - # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); - printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') - if printf_args: - match = Match(r'([\w.\->()]+)$', printf_args) - if match and match.group(1) != '__VA_ARGS__': - function_name = re.search(r'\b((?:string)?printf)\s*\(', - line, re.I).group(1) - error(filename, linenum, 'runtime/printf', 4, - 'Potential format string bug. Do %s("%%s", %s) instead.' - % (function_name, match.group(1))) - - # Check for potential memset bugs like memset(buf, sizeof(buf), 0). - match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) - if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): - error(filename, linenum, 'runtime/memset', 4, - 'Did you mean "memset(%s, 0, %s)"?' - % (match.group(1), match.group(2))) - - if Search(r'\busing namespace\b', line): - error(filename, linenum, 'build/namespaces', 5, - 'Do not use namespace using-directives. ' - 'Use using-declarations instead.') - - # Detect variable-length arrays. - match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) - if (match and match.group(2) != 'return' and match.group(2) != 'delete' and - match.group(3).find(']') == -1): - # Split the size using space and arithmetic operators as delimiters. - # If any of the resulting tokens are not compile time constants then - # report the error. - tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) - is_const = True - skip_next = False - for tok in tokens: - if skip_next: + CheckIncludeLine(filename, clean_lines, linenum, include_state, error) + return + + # Reset include state across preprocessor directives. This is meant + # to silence warnings for conditional includes. + match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) + if match: + include_state.ResetSection(match.group(1)) + + # Make Windows paths like Unix. + fullname = os.path.abspath(filename).replace('\\', '/') + + # Perform other checks now that we are sure that this is not an include line + CheckCasts(filename, clean_lines, linenum, error) + CheckGlobalStatic(filename, clean_lines, linenum, error) + CheckPrintf(filename, clean_lines, linenum, error) + + if file_extension == 'h': + # TODO(unknown): check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in CheckForNonStandardConstructs for now) + # TODO(unknown): check that classes declare or disable copy/assign + # (level 1 error) + pass + + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if Search(r'\bshort port\b', line): + if not Search(r'\bunsigned short port\b', line): + error(filename, linenum, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + else: + match = Search(r'\b(short|long(?! +double)|long long)\b', line) + if match: + error(filename, linenum, 'runtime/int', 4, + 'Use int16/int64/etc, rather than the C type %s' % + match.group(1)) + + # Check if some verboten operator overloading is going on + # TODO(unknown): catch out-of-line unary operator&: + # class X {}; + # int operator&(const X& x) { return 42; } // unary operator& + # The trick is it's hard to tell apart from binary operator&: + # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& + if Search(r'\boperator\s*&\s*\(\s*\)', line): + error(filename, linenum, 'runtime/operator', 4, + 'Unary operator& is dangerous. Do not use it.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if Search(r'\}\s*if\s*\(', line): + error(filename, linenum, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + # TODO(unknown): Catch the following case. Need to change the calling + # convention of the whole function to process multiple line to handle it. + # printf( + # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); + printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') + if printf_args: + match = Match(r'([\w.\->()]+)$', printf_args) + if match and match.group(1) != '__VA_ARGS__': + function_name = re.search(r'\b((?:string)?printf)\s*\(', line, + re.I).group(1) + error(filename, linenum, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' % + (function_name, match.group(1))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): + error(filename, linenum, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' % + (match.group(1), match.group(2))) + + if Search(r'\busing namespace\b', line): + error(filename, linenum, 'build/namespaces', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + + # Detect variable-length arrays. + match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (match and match.group(2) != 'return' and match.group(2) != 'delete' and + match.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) + is_const = True skip_next = False - continue - - if Search(r'sizeof\(.+\)', tok): continue - if Search(r'arraysize\(\w+\)', tok): continue - - tok = tok.lstrip('(') - tok = tok.rstrip(')') - if not tok: continue - if Match(r'\d+', tok): continue - if Match(r'0[xX][0-9a-fA-F]+', tok): continue - if Match(r'k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue - # A catch all for tricky sizeof cases, including 'sizeof expression', - # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' - # requires skipping the next token because we split on ' ' and '*'. - if tok.startswith('sizeof'): - skip_next = True - continue - is_const = False - break - if not is_const: - error(filename, linenum, 'runtime/arrays', 1, - 'Do not use variable-length arrays. Use an appropriately named ' - "('k' followed by CamelCase) compile-time constant for the size.") - - # Check for use of unnamed namespaces in header files. Registration - # macros are typically OK, so we allow use of "namespace {" on lines - # that end with backslashes. - if (file_extension == 'h' - and Search(r'\bnamespace\s*{', line) - and line[-1] != '\\'): - error(filename, linenum, 'build/namespaces', 4, - 'Do not use unnamed namespaces in header files. See ' - 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' - ' for more information.') + for tok in tokens: + if skip_next: + skip_next = False + continue + + if Search(r'sizeof\(.+\)', tok): continue + if Search(r'arraysize\(\w+\)', tok): continue + + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: continue + if Match(r'\d+', tok): continue + if Match(r'0[xX][0-9a-fA-F]+', tok): continue + if Match(r'k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue + # A catch all for tricky sizeof cases, including 'sizeof expression', + # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' + # requires skipping the next token because we split on ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error( + filename, linenum, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size." + ) + + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (file_extension == 'h' and Search(r'\bnamespace\s*{', line) and + line[-1] != '\\'): + error( + filename, linenum, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' + ' for more information.') def CheckGlobalStatic(filename, clean_lines, linenum, error): - """Check for unsafe global or static objects. + """Check for unsafe global or static objects. Args: filename: The name of the current file. @@ -4934,51 +4967,50 @@ def CheckGlobalStatic(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Match two lines at a time to support multiline declarations - if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): - line += clean_lines.elided[linenum + 1].strip() - - # Check for people declaring static/global STL strings at the top level. - # This is dangerous because the C++ language does not guarantee that - # globals with constructors are initialized before the first access. - match = Match( - r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', - line) - - # Remove false positives: - # - String pointers (as opposed to values). - # string *pointer - # const string *pointer - # string const *pointer - # string *const pointer - # - # - Functions and template specializations. - # string Function(... - # string Class::Method(... - # - # - Operators. These are matched separately because operator names - # cross non-word boundaries, and trying to match both operators - # and functions at the same time would decrease accuracy of - # matching identifiers. - # string Class::operator*() - if (match and - not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and - not Search(r'\boperator\W', line) and - not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))): - error(filename, linenum, 'runtime/string', 4, - 'For a static/global string constant, use a C style string instead: ' - '"%schar %s[]".' % - (match.group(1), match.group(2))) - - if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): - error(filename, linenum, 'runtime/init', 4, - 'You seem to be initializing a member variable with itself.') + line = clean_lines.elided[linenum] + + # Match two lines at a time to support multiline declarations + if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): + line += clean_lines.elided[linenum + 1].strip() + + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access. + match = Match(r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', + line) + + # Remove false positives: + # - String pointers (as opposed to values). + # string *pointer + # const string *pointer + # string const *pointer + # string *const pointer + # + # - Functions and template specializations. + # string Function(... + # string Class::Method(... + # + # - Operators. These are matched separately because operator names + # cross non-word boundaries, and trying to match both operators + # and functions at the same time would decrease accuracy of + # matching identifiers. + # string Class::operator*() + if (match and + not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and + not Search(r'\boperator\W', line) and not Match( + r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))): + error( + filename, linenum, 'runtime/string', 4, + 'For a static/global string constant, use a C style string instead: ' + '"%schar %s[]".' % (match.group(1), match.group(2))) + + if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): + error(filename, linenum, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') def CheckPrintf(filename, clean_lines, linenum, error): - """Check for printf related issues. + """Check for printf related issues. Args: filename: The name of the current file. @@ -4986,28 +5018,28 @@ def CheckPrintf(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # When snprintf is used, the second argument shouldn't be a literal. - match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) - if match and match.group(2) != '0': - # If 2nd arg is zero, snprintf is used to calculate size. - error(filename, linenum, 'runtime/printf', 3, - 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' - 'to snprintf.' % (match.group(1), match.group(2))) - - # Check if some verboten C functions are being used. - if Search(r'\bsprintf\s*\(', line): - error(filename, linenum, 'runtime/printf', 5, - 'Never use sprintf. Use snprintf instead.') - match = Search(r'\b(strcpy|strcat)\s*\(', line) - if match: - error(filename, linenum, 'runtime/printf', 4, - 'Almost always, snprintf is better than %s' % match.group(1)) + line = clean_lines.elided[linenum] + + # When snprintf is used, the second argument shouldn't be a literal. + match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if match and match.group(2) != '0': + # If 2nd arg is zero, snprintf is used to calculate size. + error(filename, linenum, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (match.group(1), match.group(2))) + + # Check if some verboten C functions are being used. + if Search(r'\bsprintf\s*\(', line): + error(filename, linenum, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + match = Search(r'\b(strcpy|strcat)\s*\(', line) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % match.group(1)) def IsDerivedFunction(clean_lines, linenum): - """Check if current line contains an inherited function. + """Check if current line contains an inherited function. Args: clean_lines: A CleansedLines instance containing the file. @@ -5016,20 +5048,20 @@ def IsDerivedFunction(clean_lines, linenum): True if current line contains a function with "override" virt-specifier. """ - # Scan back a few lines for start of current function - for i in xrange(linenum, max(-1, linenum - 10), -1): - match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) - if match: - # Look for "override" after the matching closing parenthesis - line, _, closing_paren = CloseExpression( - clean_lines, i, len(match.group(1))) - return (closing_paren >= 0 and - Search(r'\boverride\b', line[closing_paren:])) - return False + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) + if match: + # Look for "override" after the matching closing parenthesis + line, _, closing_paren = CloseExpression(clean_lines, i, + len(match.group(1))) + return (closing_paren >= 0 and + Search(r'\boverride\b', line[closing_paren:])) + return False def IsOutOfLineMethodDefinition(clean_lines, linenum): - """Check if current line contains an out-of-line method definition. + """Check if current line contains an out-of-line method definition. Args: clean_lines: A CleansedLines instance containing the file. @@ -5037,15 +5069,16 @@ def IsOutOfLineMethodDefinition(clean_lines, linenum): Returns: True if current line contains an out-of-line method definition. """ - # Scan back a few lines for start of current function - for i in xrange(linenum, max(-1, linenum - 10), -1): - if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): - return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None - return False + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): + return Match(r'^[^()]*\w+::\w+\(', + clean_lines.elided[i]) is not None + return False def IsInitializerList(clean_lines, linenum): - """Check if current line is inside constructor initializer list. + """Check if current line is inside constructor initializer list. Args: clean_lines: A CleansedLines instance containing the file. @@ -5054,41 +5087,41 @@ def IsInitializerList(clean_lines, linenum): True if current line appears to be inside constructor initializer list, False otherwise. """ - for i in xrange(linenum, 1, -1): - line = clean_lines.elided[i] - if i == linenum: - remove_function_body = Match(r'^(.*)\{\s*$', line) - if remove_function_body: - line = remove_function_body.group(1) - - if Search(r'\s:\s*\w+[({]', line): - # A lone colon tend to indicate the start of a constructor - # initializer list. It could also be a ternary operator, which - # also tend to appear in constructor initializer lists as - # opposed to parameter lists. - return True - if Search(r'\}\s*,\s*$', line): - # A closing brace followed by a comma is probably the end of a - # brace-initialized member in constructor initializer list. - return True - if Search(r'[{};]\s*$', line): - # Found one of the following: - # - A closing brace or semicolon, probably the end of the previous - # function. - # - An opening brace, probably the start of current class or namespace. - # - # Current line is probably not inside an initializer list since - # we saw one of those things without seeing the starting colon. - return False - - # Got to the beginning of the file without seeing the start of - # constructor initializer list. - return False - - -def CheckForNonConstReference(filename, clean_lines, linenum, - nesting_state, error): - """Check for non-const references. + for i in xrange(linenum, 1, -1): + line = clean_lines.elided[i] + if i == linenum: + remove_function_body = Match(r'^(.*)\{\s*$', line) + if remove_function_body: + line = remove_function_body.group(1) + + if Search(r'\s:\s*\w+[({]', line): + # A lone colon tend to indicate the start of a constructor + # initializer list. It could also be a ternary operator, which + # also tend to appear in constructor initializer lists as + # opposed to parameter lists. + return True + if Search(r'\}\s*,\s*$', line): + # A closing brace followed by a comma is probably the end of a + # brace-initialized member in constructor initializer list. + return True + if Search(r'[{};]\s*$', line): + # Found one of the following: + # - A closing brace or semicolon, probably the end of the previous + # function. + # - An opening brace, probably the start of current class or namespace. + # + # Current line is probably not inside an initializer list since + # we saw one of those things without seeing the starting colon. + return False + + # Got to the beginning of the file without seeing the start of + # constructor initializer list. + return False + + +def CheckForNonConstReference(filename, clean_lines, linenum, nesting_state, + error): + """Check for non-const references. Separate from CheckLanguage since it scans backwards from current line, instead of scanning forward. @@ -5101,131 +5134,131 @@ def CheckForNonConstReference(filename, clean_lines, linenum, the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # Do nothing if there is no '&' on current line. - line = clean_lines.elided[linenum] - if '&' not in line: - return - - # If a function is inherited, current function doesn't have much of - # a choice, so any non-const references should not be blamed on - # derived function. - if IsDerivedFunction(clean_lines, linenum): - return - - # Don't warn on out-of-line method definitions, as we would warn on the - # in-line declaration, if it isn't marked with 'override'. - if IsOutOfLineMethodDefinition(clean_lines, linenum): - return - - # Long type names may be broken across multiple lines, usually in one - # of these forms: - # LongType - # ::LongTypeContinued &identifier - # LongType:: - # LongTypeContinued &identifier - # LongType< - # ...>::LongTypeContinued &identifier - # - # If we detected a type split across two lines, join the previous - # line to current line so that we can match const references - # accordingly. - # - # Note that this only scans back one line, since scanning back - # arbitrary number of lines would be expensive. If you have a type - # that spans more than 2 lines, please use a typedef. - if linenum > 1: - previous = None - if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): - # previous_line\n + ::current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', - clean_lines.elided[linenum - 1]) - elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): - # previous_line::\n + current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', - clean_lines.elided[linenum - 1]) - if previous: - line = previous.group(1) + line.lstrip() - else: - # Check for templated parameter that is split across multiple lines - endpos = line.rfind('>') - if endpos > -1: - (_, startline, startpos) = ReverseCloseExpression( - clean_lines, linenum, endpos) - if startpos > -1 and startline < linenum: - # Found the matching < on an earlier line, collect all - # pieces up to current line. - line = '' - for i in xrange(startline, linenum + 1): - line += clean_lines.elided[i].strip() - - # Check for non-const references in function parameters. A single '&' may - # found in the following places: - # inside expression: binary & for bitwise AND - # inside expression: unary & for taking the address of something - # inside declarators: reference parameter - # We will exclude the first two cases by checking that we are not inside a - # function body, including one that was just introduced by a trailing '{'. - # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. - if (nesting_state.previous_stack_top and - not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or - isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): - # Not at toplevel, not within a class, and not within a namespace - return - - # Avoid initializer lists. We only need to scan back from the - # current line for something that starts with ':'. - # - # We don't need to check the current line, since the '&' would - # appear inside the second set of parentheses on the current line as - # opposed to the first set. - if linenum > 0: - for i in xrange(linenum - 1, max(0, linenum - 10), -1): - previous_line = clean_lines.elided[i] - if not Search(r'[),]\s*$', previous_line): - break - if Match(r'^\s*:\s+\S', previous_line): + # Do nothing if there is no '&' on current line. + line = clean_lines.elided[linenum] + if '&' not in line: + return + + # If a function is inherited, current function doesn't have much of + # a choice, so any non-const references should not be blamed on + # derived function. + if IsDerivedFunction(clean_lines, linenum): + return + + # Don't warn on out-of-line method definitions, as we would warn on the + # in-line declaration, if it isn't marked with 'override'. + if IsOutOfLineMethodDefinition(clean_lines, linenum): + return + + # Long type names may be broken across multiple lines, usually in one + # of these forms: + # LongType + # ::LongTypeContinued &identifier + # LongType:: + # LongTypeContinued &identifier + # LongType< + # ...>::LongTypeContinued &identifier + # + # If we detected a type split across two lines, join the previous + # line to current line so that we can match const references + # accordingly. + # + # Note that this only scans back one line, since scanning back + # arbitrary number of lines would be expensive. If you have a type + # that spans more than 2 lines, please use a typedef. + if linenum > 1: + previous = None + if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): + # previous_line\n + ::current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', + clean_lines.elided[linenum - 1]) + elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): + # previous_line::\n + current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', + clean_lines.elided[linenum - 1]) + if previous: + line = previous.group(1) + line.lstrip() + else: + # Check for templated parameter that is split across multiple lines + endpos = line.rfind('>') + if endpos > -1: + (_, startline, startpos) = ReverseCloseExpression( + clean_lines, linenum, endpos) + if startpos > -1 and startline < linenum: + # Found the matching < on an earlier line, collect all + # pieces up to current line. + line = '' + for i in xrange(startline, linenum + 1): + line += clean_lines.elided[i].strip() + + # Check for non-const references in function parameters. A single '&' may + # found in the following places: + # inside expression: binary & for bitwise AND + # inside expression: unary & for taking the address of something + # inside declarators: reference parameter + # We will exclude the first two cases by checking that we are not inside a + # function body, including one that was just introduced by a trailing '{'. + # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. + if (nesting_state.previous_stack_top and + not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or + isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): + # Not at toplevel, not within a class, and not within a namespace + return + + # Avoid initializer lists. We only need to scan back from the + # current line for something that starts with ':'. + # + # We don't need to check the current line, since the '&' would + # appear inside the second set of parentheses on the current line as + # opposed to the first set. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 10), -1): + previous_line = clean_lines.elided[i] + if not Search(r'[),]\s*$', previous_line): + break + if Match(r'^\s*:\s+\S', previous_line): + return + + # Avoid preprocessors + if Search(r'\\\s*$', line): return - # Avoid preprocessors - if Search(r'\\\s*$', line): - return - - # Avoid constructor initializer lists - if IsInitializerList(clean_lines, linenum): - return - - # We allow non-const references in a few standard places, like functions - # called "swap()" or iostream operators like "<<" or ">>". Do not check - # those function parameters. - # - # We also accept & in static_assert, which looks like a function but - # it's actually a declaration expression. - whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' - r'operator\s*[<>][<>]|' - r'static_assert|COMPILE_ASSERT' - r')\s*\(') - if Search(whitelisted_functions, line): - return - elif not Search(r'\S+\([^)]*$', line): - # Don't see a whitelisted function on this line. Actually we - # didn't see any function name on this line, so this is likely a - # multi-line parameter list. Try a bit harder to catch this case. - for i in xrange(2): - if (linenum > i and - Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): + # Avoid constructor initializer lists + if IsInitializerList(clean_lines, linenum): return - decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body - for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): - if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter): - error(filename, linenum, 'runtime/references', 2, - 'Is this a non-const reference? ' - 'If so, make const or use a pointer: ' + - ReplaceAll(' *<', '<', parameter)) + # We allow non-const references in a few standard places, like functions + # called "swap()" or iostream operators like "<<" or ">>". Do not check + # those function parameters. + # + # We also accept & in static_assert, which looks like a function but + # it's actually a declaration expression. + whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' + r'operator\s*[<>][<>]|' + r'static_assert|COMPILE_ASSERT' + r')\s*\(') + if Search(whitelisted_functions, line): + return + elif not Search(r'\S+\([^)]*$', line): + # Don't see a whitelisted function on this line. Actually we + # didn't see any function name on this line, so this is likely a + # multi-line parameter list. Try a bit harder to catch this case. + for i in xrange(2): + if (linenum > i and Search(whitelisted_functions, + clean_lines.elided[linenum - i - 1])): + return + + decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body + for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): + if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter): + error(filename, linenum, 'runtime/references', 2, + 'Is this a non-const reference? ' + 'If so, make const or use a pointer: ' + ReplaceAll( + ' *<', '<', parameter)) def CheckCasts(filename, clean_lines, linenum, error): - """Various cast related checks. + """Various cast related checks. Args: filename: The name of the current file. @@ -5233,118 +5266,116 @@ def CheckCasts(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Check to see if they're using an conversion function cast. - # I just try to capture the most common basic types, though there are more. - # Parameterless conversion functions, such as bool(), are allowed as they are - # probably a member operator declaration or default constructor. - match = Search( - r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b' - r'(int|float|double|bool|char|int32|uint32|int64|uint64)' - r'(\([^)].*)', line) - expecting_function = ExpectingFunctionArgs(clean_lines, linenum) - if match and not expecting_function: - matched_type = match.group(2) - - # matched_new_or_template is used to silence two false positives: - # - New operators - # - Template arguments with function types + line = clean_lines.elided[linenum] + + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they are + # probably a member operator declaration or default constructor. + match = Search(r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b' + r'(int|float|double|bool|char|int32|uint32|int64|uint64)' + r'(\([^)].*)', line) + expecting_function = ExpectingFunctionArgs(clean_lines, linenum) + if match and not expecting_function: + matched_type = match.group(2) + + # matched_new_or_template is used to silence two false positives: + # - New operators + # - Template arguments with function types + # + # For template arguments, we match on types immediately following + # an opening bracket without any spaces. This is a fast way to + # silence the common case where the function type is the first + # template argument. False negative with less-than comparison is + # avoided because those operators are usually followed by a space. + # + # function // bracket + no space = false positive + # value < double(42) // bracket + space = true positive + matched_new_or_template = match.group(1) + + # Avoid arrays by looking for brackets that come after the closing + # parenthesis. + if Match(r'\([^()]+\)\s*\[', match.group(3)): + return + + # Other things to ignore: + # - Function pointers + # - Casts to pointer types + # - Placement new + # - Alias declarations + matched_funcptr = match.group(3) + if (matched_new_or_template is None and not (matched_funcptr and (Match( + r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', + matched_funcptr) or matched_funcptr.startswith('(*)'))) and + not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and + not Search(r'new\(\S+\)\s*' + matched_type, line)): + error(filename, linenum, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % matched_type) + + if not expecting_function: + CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', + error) + + # This doesn't catch all cases. Consider (const char * const)"hello". # - # For template arguments, we match on types immediately following - # an opening bracket without any spaces. This is a fast way to - # silence the common case where the function type is the first - # template argument. False negative with less-than comparison is - # avoided because those operators are usually followed by a space. + # (char *) "foo" should always be a const_cast (reinterpret_cast won't + # compile). + if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', + r'\((char\s?\*+\s?)\)\s*"', error): + pass + else: + # Check pointer casts for other than string constants + CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', + r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. # - # function // bracket + no space = false positive - # value < double(42) // bracket + space = true positive - matched_new_or_template = match.group(1) - - # Avoid arrays by looking for brackets that come after the closing - # parenthesis. - if Match(r'\([^()]+\)\s*\[', match.group(3)): - return - - # Other things to ignore: - # - Function pointers - # - Casts to pointer types - # - Placement new - # - Alias declarations - matched_funcptr = match.group(3) - if (matched_new_or_template is None and - not (matched_funcptr and - (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', - matched_funcptr) or - matched_funcptr.startswith('(*)'))) and - not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and - not Search(r'new\(\S+\)\s*' + matched_type, line)): - error(filename, linenum, 'readability/casting', 4, - 'Using deprecated casting style. ' - 'Use static_cast<%s>(...) instead' % - matched_type) - - if not expecting_function: - CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', - r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) - - # This doesn't catch all cases. Consider (const char * const)"hello". - # - # (char *) "foo" should always be a const_cast (reinterpret_cast won't - # compile). - if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', - r'\((char\s?\*+\s?)\)\s*"', error): - pass - else: - # Check pointer casts for other than string constants - CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', - r'\((\w+\s?\*+\s?)\)', error) - - # In addition, we look for people taking the address of a cast. This - # is dangerous -- casts can assign to temporaries, so the pointer doesn't - # point where you think. - # - # Some non-identifier character is required before the '&' for the - # expression to be recognized as a cast. These are casts: - # expression = &static_cast(temporary()); - # function(&(int*)(temporary())); - # - # This is not a cast: - # reference_type&(int* function_param); - match = Search( - r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' - r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) - if match: - # Try a better error message when the & is bound to something - # dereferenced by the casted pointer, as opposed to the casted - # pointer itself. - parenthesis_error = False - match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line) + # Some non-identifier character is required before the '&' for the + # expression to be recognized as a cast. These are casts: + # expression = &static_cast(temporary()); + # function(&(int*)(temporary())); + # + # This is not a cast: + # reference_type&(int* function_param); + match = Search(r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' + r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) if match: - _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1))) - if x1 >= 0 and clean_lines.elided[y1][x1] == '(': - _, y2, x2 = CloseExpression(clean_lines, y1, x1) - if x2 >= 0: - extended_line = clean_lines.elided[y2][x2:] - if y2 < clean_lines.NumLines() - 1: - extended_line += clean_lines.elided[y2 + 1] - if Match(r'\s*(?:->|\[)', extended_line): - parenthesis_error = True - - if parenthesis_error: - error(filename, linenum, 'readability/casting', 4, - ('Are you taking an address of something dereferenced ' - 'from a cast? Wrapping the dereferenced expression in ' - 'parentheses will make the binding more obvious')) - else: - error(filename, linenum, 'runtime/casting', 4, - ('Are you taking an address of a cast? ' - 'This is dangerous: could be a temp var. ' - 'Take the address before doing the cast, rather than after')) + # Try a better error message when the & is bound to something + # dereferenced by the casted pointer, as opposed to the casted + # pointer itself. + parenthesis_error = False + match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', + line) + if match: + _, y1, x1 = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if x1 >= 0 and clean_lines.elided[y1][x1] == '(': + _, y2, x2 = CloseExpression(clean_lines, y1, x1) + if x2 >= 0: + extended_line = clean_lines.elided[y2][x2:] + if y2 < clean_lines.NumLines() - 1: + extended_line += clean_lines.elided[y2 + 1] + if Match(r'\s*(?:->|\[)', extended_line): + parenthesis_error = True + + if parenthesis_error: + error(filename, linenum, 'readability/casting', 4, + ('Are you taking an address of something dereferenced ' + 'from a cast? Wrapping the dereferenced expression in ' + 'parentheses will make the binding more obvious')) + else: + error(filename, linenum, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): - """Checks for a C-style cast by looking for the pattern. + """Checks for a C-style cast by looking for the pattern. Args: filename: The name of the current file. @@ -5359,96 +5390,96 @@ def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): True if an error was emitted. False otherwise. """ - line = clean_lines.elided[linenum] - match = Search(pattern, line) - if not match: - return False + line = clean_lines.elided[linenum] + match = Search(pattern, line) + if not match: + return False - # Exclude lines with keywords that tend to look like casts - context = line[0:match.start(1) - 1] - if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): - return False + # Exclude lines with keywords that tend to look like casts + context = line[0:match.start(1) - 1] + if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): + return False - # Try expanding current context to see if we one level of - # parentheses inside a macro. - if linenum > 0: - for i in xrange(linenum - 1, max(0, linenum - 5), -1): - context = clean_lines.elided[i] + context - if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): - return False + # Try expanding current context to see if we one level of + # parentheses inside a macro. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 5), -1): + context = clean_lines.elided[i] + context + if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): + return False - # operator++(int) and operator--(int) - if context.endswith(' operator++') or context.endswith(' operator--'): - return False + # operator++(int) and operator--(int) + if context.endswith(' operator++') or context.endswith(' operator--'): + return False - # A single unnamed argument for a function tends to look like old - # style cast. If we see those, don't issue warnings for deprecated - # casts, instead issue warnings for unnamed arguments where - # appropriate. - # - # These are things that we want warnings for, since the style guide - # explicitly require all parameters to be named: - # Function(int); - # Function(int) { - # ConstMember(int) const; - # ConstMember(int) const { - # ExceptionMember(int) throw (...); - # ExceptionMember(int) throw (...) { - # PureVirtual(int) = 0; - # [](int) -> bool { - # - # These are functions of some sort, where the compiler would be fine - # if they had named parameters, but people often omit those - # identifiers to reduce clutter: - # (FunctionPointer)(int); - # (FunctionPointer)(int) = value; - # Function((function_pointer_arg)(int)) - # Function((function_pointer_arg)(int), int param) - # ; - # <(FunctionPointerTemplateArgument)(int)>; - remainder = line[match.end(0):] - if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', - remainder): - # Looks like an unnamed parameter. - - # Don't warn on any kind of template arguments. - if Match(r'^\s*>', remainder): - return False - - # Don't warn on assignments to function pointers, but keep warnings for - # unnamed parameters to pure virtual functions. Note that this pattern - # will also pass on assignments of "0" to function pointers, but the - # preferred values for those would be "nullptr" or "NULL". - matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder) - if matched_zero and matched_zero.group(1) != '0': - return False - - # Don't warn on function pointer declarations. For this we need - # to check what came before the "(type)" string. - if Match(r'.*\)\s*$', line[0:match.start(0)]): - return False - - # Don't warn if the parameter is named with block comments, e.g.: - # Function(int /*unused_param*/); - raw_line = clean_lines.raw_lines[linenum] - if '/*' in raw_line: - return False - - # Passed all filters, issue warning here. - error(filename, linenum, 'readability/function', 3, - 'All parameters should be named in a function') - return True + # A single unnamed argument for a function tends to look like old + # style cast. If we see those, don't issue warnings for deprecated + # casts, instead issue warnings for unnamed arguments where + # appropriate. + # + # These are things that we want warnings for, since the style guide + # explicitly require all parameters to be named: + # Function(int); + # Function(int) { + # ConstMember(int) const; + # ConstMember(int) const { + # ExceptionMember(int) throw (...); + # ExceptionMember(int) throw (...) { + # PureVirtual(int) = 0; + # [](int) -> bool { + # + # These are functions of some sort, where the compiler would be fine + # if they had named parameters, but people often omit those + # identifiers to reduce clutter: + # (FunctionPointer)(int); + # (FunctionPointer)(int) = value; + # Function((function_pointer_arg)(int)) + # Function((function_pointer_arg)(int), int param) + # ; + # <(FunctionPointerTemplateArgument)(int)>; + remainder = line[match.end(0):] + if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', + remainder): + # Looks like an unnamed parameter. + + # Don't warn on any kind of template arguments. + if Match(r'^\s*>', remainder): + return False - # At this point, all that should be left is actual casts. - error(filename, linenum, 'readability/casting', 4, - 'Using C-style cast. Use %s<%s>(...) instead' % - (cast_type, match.group(1))) + # Don't warn on assignments to function pointers, but keep warnings for + # unnamed parameters to pure virtual functions. Note that this pattern + # will also pass on assignments of "0" to function pointers, but the + # preferred values for those would be "nullptr" or "NULL". + matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder) + if matched_zero and matched_zero.group(1) != '0': + return False - return True + # Don't warn on function pointer declarations. For this we need + # to check what came before the "(type)" string. + if Match(r'.*\)\s*$', line[0:match.start(0)]): + return False + + # Don't warn if the parameter is named with block comments, e.g.: + # Function(int /*unused_param*/); + raw_line = clean_lines.raw_lines[linenum] + if '/*' in raw_line: + return False + + # Passed all filters, issue warning here. + error(filename, linenum, 'readability/function', 3, + 'All parameters should be named in a function') + return True + + # At this point, all that should be left is actual casts. + error(filename, linenum, 'readability/casting', 4, + 'Using C-style cast. Use %s<%s>(...) instead' % + (cast_type, match.group(1))) + + return True def ExpectingFunctionArgs(clean_lines, linenum): - """Checks whether where function type arguments are expected. + """Checks whether where function type arguments are expected. Args: clean_lines: A CleansedLines instance containing the file. @@ -5458,78 +5489,107 @@ def ExpectingFunctionArgs(clean_lines, linenum): True if the line at 'linenum' is inside something that expects arguments of function types. """ - line = clean_lines.elided[linenum] - return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or - (linenum >= 2 and - (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', - clean_lines.elided[linenum - 1]) or - Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', - clean_lines.elided[linenum - 2]) or - Search(r'\bstd::m?function\s*\<\s*$', - clean_lines.elided[linenum - 1])))) + line = clean_lines.elided[linenum] + return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or + (linenum >= 2 and + (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', + clean_lines.elided[linenum - 1]) or + Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', + clean_lines.elided[linenum - 2]) or + Search(r'\bstd::m?function\s*\<\s*$', + clean_lines.elided[linenum - 1])))) _HEADERS_CONTAINING_TEMPLATES = ( - ('', ('deque',)), - ('', ('unary_function', 'binary_function', - 'plus', 'minus', 'multiplies', 'divides', 'modulus', - 'negate', - 'equal_to', 'not_equal_to', 'greater', 'less', - 'greater_equal', 'less_equal', - 'logical_and', 'logical_or', 'logical_not', - 'unary_negate', 'not1', 'binary_negate', 'not2', - 'bind1st', 'bind2nd', - 'pointer_to_unary_function', - 'pointer_to_binary_function', - 'ptr_fun', - 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', - 'mem_fun_ref_t', - 'const_mem_fun_t', 'const_mem_fun1_t', - 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', - 'mem_fun_ref', - )), - ('', ('numeric_limits',)), - ('', ('list',)), - ('', ('map', 'multimap',)), - ('', ('allocator',)), - ('', ('queue', 'priority_queue',)), - ('', ('set', 'multiset',)), - ('', ('stack',)), - ('', ('char_traits', 'basic_string',)), - ('', ('tuple',)), - ('', ('pair',)), - ('', ('vector',)), + ('', ('deque', )), + ('', ( + 'unary_function', + 'binary_function', + 'plus', + 'minus', + 'multiplies', + 'divides', + 'modulus', + 'negate', + 'equal_to', + 'not_equal_to', + 'greater', + 'less', + 'greater_equal', + 'less_equal', + 'logical_and', + 'logical_or', + 'logical_not', + 'unary_negate', + 'not1', + 'binary_negate', + 'not2', + 'bind1st', + 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', + 'mem_fun', + 'mem_fun1_t', + 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', + 'const_mem_fun1_t', + 'const_mem_fun_ref_t', + 'const_mem_fun1_ref_t', + 'mem_fun_ref', )), + ('', ('numeric_limits', )), + ('', ('list', )), + ('', ( + 'map', + 'multimap', )), + ('', ('allocator', )), + ('', ( + 'queue', + 'priority_queue', )), + ('', ( + 'set', + 'multiset', )), + ('', ('stack', )), + ('', ( + 'char_traits', + 'basic_string', )), + ('', ('tuple', )), + ('', ('pair', )), + ('', ('vector', )), # gcc extensions. # Note: std::hash is their hash, ::hash is our hash - ('', ('hash_map', 'hash_multimap',)), - ('', ('hash_set', 'hash_multiset',)), - ('', ('slist',)), - ) + ('', ( + 'hash_map', + 'hash_multimap', )), + ('', ( + 'hash_set', + 'hash_multiset', )), + ('', ('slist', )), ) _RE_PATTERN_STRING = re.compile(r'\bstring\b') _re_pattern_algorithm_header = [] for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap', 'transform'): - # Match max(..., ...), max(..., ...), but not foo->max, foo.max or - # type::max(). - _re_pattern_algorithm_header.append( - (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), - _template, - '')) + # Match max(..., ...), max(..., ...), but not foo->max, foo.max or + # type::max(). + _re_pattern_algorithm_header.append( + (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), _template, + '')) _re_pattern_templates = [] for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: - for _template in _templates: - _re_pattern_templates.append( - (re.compile(r'(\<|\b)' + _template + r'\s*\<'), - _template + '<>', - _header)) + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), _template + '<>', + _header)) def FilesBelongToSameModule(filename_cc, filename_h): - """Check if these two filenames belong to the same module. + """Check if these two filenames belong to the same module. The concept of a 'module' here is a as follows: foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the @@ -5558,33 +5618,33 @@ def FilesBelongToSameModule(filename_cc, filename_h): string: the additional prefix needed to open the header file. """ - if not filename_cc.endswith('.cc'): - return (False, '') - filename_cc = filename_cc[:-len('.cc')] - if filename_cc.endswith('_unittest'): - filename_cc = filename_cc[:-len('_unittest')] - elif filename_cc.endswith('_test'): - filename_cc = filename_cc[:-len('_test')] - filename_cc = filename_cc.replace('/public/', '/') - filename_cc = filename_cc.replace('/internal/', '/') - - if not filename_h.endswith('.h'): - return (False, '') - filename_h = filename_h[:-len('.h')] - if filename_h.endswith('-inl'): - filename_h = filename_h[:-len('-inl')] - filename_h = filename_h.replace('/public/', '/') - filename_h = filename_h.replace('/internal/', '/') - - files_belong_to_same_module = filename_cc.endswith(filename_h) - common_path = '' - if files_belong_to_same_module: - common_path = filename_cc[:-len(filename_h)] - return files_belong_to_same_module, common_path + if not filename_cc.endswith('.cc'): + return (False, '') + filename_cc = filename_cc[:-len('.cc')] + if filename_cc.endswith('_unittest'): + filename_cc = filename_cc[:-len('_unittest')] + elif filename_cc.endswith('_test'): + filename_cc = filename_cc[:-len('_test')] + filename_cc = filename_cc.replace('/public/', '/') + filename_cc = filename_cc.replace('/internal/', '/') + + if not filename_h.endswith('.h'): + return (False, '') + filename_h = filename_h[:-len('.h')] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') + + files_belong_to_same_module = filename_cc.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cc[:-len(filename_h)] + return files_belong_to_same_module, common_path def UpdateIncludeState(filename, include_dict, io=codecs): - """Fill up the include_dict with new includes found from the file. + """Fill up the include_dict with new includes found from the file. Args: filename: the name of the header to read. @@ -5594,25 +5654,28 @@ def UpdateIncludeState(filename, include_dict, io=codecs): Returns: True if a header was successfully added. False otherwise. """ - headerfile = None - try: - headerfile = io.open(filename, 'r', 'utf8', 'replace') - except IOError: - return False - linenum = 0 - for line in headerfile: - linenum += 1 - clean_line = CleanseComments(line) - match = _RE_PATTERN_INCLUDE.search(clean_line) - if match: - include = match.group(2) - include_dict.setdefault(include, linenum) - return True + headerfile = None + try: + headerfile = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + linenum = 0 + for line in headerfile: + linenum += 1 + clean_line = CleanseComments(line) + match = _RE_PATTERN_INCLUDE.search(clean_line) + if match: + include = match.group(2) + include_dict.setdefault(include, linenum) + return True -def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, +def CheckForIncludeWhatYouUse(filename, + clean_lines, + include_state, + error, io=codecs): - """Reports for missing stl includes. + """Reports for missing stl includes. This function will output warnings to make sure you are including the headers necessary for the stl containers and functions that you use. We only give one @@ -5628,87 +5691,88 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, io: The IO factory to use to read the header file. Provided for unittest injection. """ - required = {} # A map of header name to linenumber and the template entity. - # Example of required: { '': (1219, 'less<>') } + required = {} # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } - for linenum in xrange(clean_lines.NumLines()): - line = clean_lines.elided[linenum] - if not line or line[0] == '#': - continue + for linenum in xrange(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue - # String is special -- it is a non-templatized type in STL. - matched = _RE_PATTERN_STRING.search(line) - if matched: - # Don't warn about strings in non-STL namespaces: - # (We check only the first match per line; good enough.) - prefix = line[:matched.start()] - if prefix.endswith('std::') or not prefix.endswith('::'): - required[''] = (linenum, 'string') - - for pattern, template, header in _re_pattern_algorithm_header: - if pattern.search(line): - required[header] = (linenum, template) - - # The following function is just a speed up, no semantics are changed. - if not '<' in line: # Reduces the cpu time usage by skipping lines. - continue - - for pattern, template, header in _re_pattern_templates: - if pattern.search(line): - required[header] = (linenum, template) - - # The policy is that if you #include something in foo.h you don't need to - # include it again in foo.cc. Here, we will look at possible includes. - # Let's flatten the include_state include_list and copy it into a dictionary. - include_dict = dict([item for sublist in include_state.include_list - for item in sublist]) - - # Did we find the header for this file (if any) and successfully load it? - header_found = False - - # Use the absolute path so that matching works properly. - abs_filename = FileInfo(filename).FullName() - - # For Emacs's flymake. - # If cpplint is invoked from Emacs's flymake, a temporary file is generated - # by flymake and that file name might end with '_flymake.cc'. In that case, - # restore original file name here so that the corresponding header file can be - # found. - # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' - # instead of 'foo_flymake.h' - abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) - - # include_dict is modified during iteration, so we iterate over a copy of - # the keys. - header_keys = include_dict.keys() - for header in header_keys: - (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) - fullpath = common_path + header - if same_module and UpdateIncludeState(fullpath, include_dict, io): - header_found = True - - # If we can't find the header file for a .cc, assume it's because we don't - # know where to look. In that case we'll give up as we're not sure they - # didn't include it in the .h file. - # TODO(unknown): Do a better job of finding .h files so we are confident that - # not having the .h file means there isn't one. - if filename.endswith('.cc') and not header_found: - return - - # All the lines have been processed, report the errors found. - for required_header_unstripped in required: - template = required[required_header_unstripped][1] - if required_header_unstripped.strip('<>"') not in include_dict: - error(filename, required[required_header_unstripped][0], - 'build/include_what_you_use', 4, - 'Add #include ' + required_header_unstripped + ' for ' + template) + # String is special -- it is a non-templatized type in STL. + matched = _RE_PATTERN_STRING.search(line) + if matched: + # Don't warn about strings in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[''] = (linenum, 'string') + + for pattern, template, header in _re_pattern_algorithm_header: + if pattern.search(line): + required[header] = (linenum, template) + + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue + + for pattern, template, header in _re_pattern_templates: + if pattern.search(line): + required[header] = (linenum, template) + + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cc. Here, we will look at possible includes. + # Let's flatten the include_state include_list and copy it into a dictionary. + include_dict = dict( + [item for sublist in include_state.include_list for item in sublist]) + + # Did we find the header for this file (if any) and successfully load it? + header_found = False + + # Use the absolute path so that matching works properly. + abs_filename = FileInfo(filename).FullName() + + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cc'. In that case, + # restore original file name here so that the corresponding header file can be + # found. + # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' + # instead of 'foo_flymake.h' + abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) + + # include_dict is modified during iteration, so we iterate over a copy of + # the keys. + header_keys = include_dict.keys() + for header in header_keys: + (same_module, common_path) = FilesBelongToSameModule(abs_filename, + header) + fullpath = common_path + header + if same_module and UpdateIncludeState(fullpath, include_dict, io): + header_found = True + + # If we can't find the header file for a .cc, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # TODO(unknown): Do a better job of finding .h files so we are confident that + # not having the .h file means there isn't one. + if filename.endswith('.cc') and not header_found: + return + + # All the lines have been processed, report the errors found. + for required_header_unstripped in required: + template = required[required_header_unstripped][1] + if required_header_unstripped.strip('<>"') not in include_dict: + error(filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, 'Add #include ' + + required_header_unstripped + ' for ' + template) _RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): - """Check that make_pair's template arguments are deduced. + """Check that make_pair's template arguments are deduced. G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are specified explicitly, and such use isn't intended in any case. @@ -5719,17 +5783,20 @@ def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) - if match: - error(filename, linenum, 'build/explicit_make_pair', - 4, # 4 = high confidence - 'For C++11-compatibility, omit template arguments from make_pair' - ' OR use pair directly OR if appropriate, construct a pair directly') + line = clean_lines.elided[linenum] + match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) + if match: + error( + filename, + linenum, + 'build/explicit_make_pair', + 4, # 4 = high confidence + 'For C++11-compatibility, omit template arguments from make_pair' + ' OR use pair directly OR if appropriate, construct a pair directly') def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error): - """Check that default lambda captures are not used. + """Check that default lambda captures are not used. Args: filename: The name of the current file. @@ -5737,24 +5804,28 @@ def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # A lambda introducer specifies a default capture if it starts with "[=" - # or if it starts with "[&" _not_ followed by an identifier. - match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line) - if match: - # Found a potential error, check what comes after the lambda-introducer. - # If it's not open parenthesis (for lambda-declarator) or open brace - # (for compound-statement), it's not a lambda. - line, _, pos = CloseExpression(clean_lines, linenum, len(match.group(1))) - if pos >= 0 and Match(r'^\s*[{(]', line[pos:]): - error(filename, linenum, 'build/c++11', - 4, # 4 = high confidence - 'Default lambda captures are an unapproved C++ feature.') + line = clean_lines.elided[linenum] + + # A lambda introducer specifies a default capture if it starts with "[=" + # or if it starts with "[&" _not_ followed by an identifier. + match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line) + if match: + # Found a potential error, check what comes after the lambda-introducer. + # If it's not open parenthesis (for lambda-declarator) or open brace + # (for compound-statement), it's not a lambda. + line, _, pos = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if pos >= 0 and Match(r'^\s*[{(]', line[pos:]): + error( + filename, + linenum, + 'build/c++11', + 4, # 4 = high confidence + 'Default lambda captures are an unapproved C++ feature.') def CheckRedundantVirtual(filename, clean_lines, linenum, error): - """Check if line contains a redundant "virtual" function-specifier. + """Check if line contains a redundant "virtual" function-specifier. Args: filename: The name of the current file. @@ -5762,63 +5833,64 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Look for "virtual" on current line. - line = clean_lines.elided[linenum] - virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) - if not virtual: return - - # Ignore "virtual" keywords that are near access-specifiers. These - # are only used in class base-specifier and do not apply to member - # functions. - if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or - Match(r'^\s+(public|protected|private)\b', virtual.group(3))): - return - - # Ignore the "virtual" keyword from virtual base classes. Usually - # there is a column on the same line in these cases (virtual base - # classes are rare in google3 because multiple inheritance is rare). - if Match(r'^.*[^:]:[^:].*$', line): return - - # Look for the next opening parenthesis. This is the start of the - # parameter list (possibly on the next line shortly after virtual). - # TODO(unknown): doesn't work if there are virtual functions with - # decltype() or other things that use parentheses, but csearch suggests - # that this is rare. - end_col = -1 - end_line = -1 - start_col = len(virtual.group(2)) - for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): - line = clean_lines.elided[start_line][start_col:] - parameter_list = Match(r'^([^(]*)\(', line) - if parameter_list: - # Match parentheses to find the end of the parameter list - (_, end_line, end_col) = CloseExpression( - clean_lines, start_line, start_col + len(parameter_list.group(1))) - break - start_col = 0 - - if end_col < 0: - return # Couldn't find end of parameter list, give up - - # Look for "override" or "final" after the parameter list - # (possibly on the next few lines). - for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): - line = clean_lines.elided[i][end_col:] - match = Search(r'\b(override|final)\b', line) - if match: - error(filename, linenum, 'readability/inheritance', 4, - ('"virtual" is redundant since function is ' - 'already declared as "%s"' % match.group(1))) + # Look for "virtual" on current line. + line = clean_lines.elided[linenum] + virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) + if not virtual: return + + # Ignore "virtual" keywords that are near access-specifiers. These + # are only used in class base-specifier and do not apply to member + # functions. + if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or + Match(r'^\s+(public|protected|private)\b', virtual.group(3))): + return - # Set end_col to check whole lines after we are done with the - # first line. - end_col = 0 - if Search(r'[^\w]\s*$', line): - break + # Ignore the "virtual" keyword from virtual base classes. Usually + # there is a column on the same line in these cases (virtual base + # classes are rare in google3 because multiple inheritance is rare). + if Match(r'^.*[^:]:[^:].*$', line): return + + # Look for the next opening parenthesis. This is the start of the + # parameter list (possibly on the next line shortly after virtual). + # TODO(unknown): doesn't work if there are virtual functions with + # decltype() or other things that use parentheses, but csearch suggests + # that this is rare. + end_col = -1 + end_line = -1 + start_col = len(virtual.group(2)) + for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): + line = clean_lines.elided[start_line][start_col:] + parameter_list = Match(r'^([^(]*)\(', line) + if parameter_list: + # Match parentheses to find the end of the parameter list + (_, end_line, end_col) = CloseExpression( + clean_lines, start_line, + start_col + len(parameter_list.group(1))) + break + start_col = 0 + + if end_col < 0: + return # Couldn't find end of parameter list, give up + + # Look for "override" or "final" after the parameter list + # (possibly on the next few lines). + for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): + line = clean_lines.elided[i][end_col:] + match = Search(r'\b(override|final)\b', line) + if match: + error(filename, linenum, 'readability/inheritance', 4, + ('"virtual" is redundant since function is ' + 'already declared as "%s"' % match.group(1))) + + # Set end_col to check whole lines after we are done with the + # first line. + end_col = 0 + if Search(r'[^\w]\s*$', line): + break def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): - """Check if line contains a redundant "override" or "final" virt-specifier. + """Check if line contains a redundant "override" or "final" virt-specifier. Args: filename: The name of the current file. @@ -5826,32 +5898,30 @@ def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Look for closing parenthesis nearby. We need one to confirm where - # the declarator ends and where the virt-specifier starts to avoid - # false positives. - line = clean_lines.elided[linenum] - declarator_end = line.rfind(')') - if declarator_end >= 0: - fragment = line[declarator_end:] - else: - if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: - fragment = line + # Look for closing parenthesis nearby. We need one to confirm where + # the declarator ends and where the virt-specifier starts to avoid + # false positives. + line = clean_lines.elided[linenum] + declarator_end = line.rfind(')') + if declarator_end >= 0: + fragment = line[declarator_end:] else: - return - - # Check that at most one of "override" or "final" is present, not both - if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): - error(filename, linenum, 'readability/inheritance', 4, - ('"override" is redundant since function is ' - 'already declared as "final"')) - + if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: + fragment = line + else: + return + # Check that at most one of "override" or "final" is present, not both + if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): + error(filename, linenum, 'readability/inheritance', 4, + ('"override" is redundant since function is ' + 'already declared as "final"')) # Returns true if we are at a new block, and it is directly # inside of a namespace. def IsBlockInNameSpace(nesting_state, is_forward_declaration): - """Checks that the new block is directly in a namespace. + """Checks that the new block is directly in a namespace. Args: nesting_state: The _NestingState object that contains info about our state. @@ -5859,21 +5929,21 @@ def IsBlockInNameSpace(nesting_state, is_forward_declaration): Returns: Whether or not the new block is directly in a namespace. """ - if is_forward_declaration: - if len(nesting_state.stack) >= 1 and ( - isinstance(nesting_state.stack[-1], _NamespaceInfo)): - return True - else: - return False + if is_forward_declaration: + if len(nesting_state.stack) >= 1 and ( + isinstance(nesting_state.stack[-1], _NamespaceInfo)): + return True + else: + return False - return (len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.stack[-2], _NamespaceInfo)) + return (len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.stack[-2], _NamespaceInfo)) def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, raw_lines_no_comments, linenum): - """This method determines if we should apply our namespace indentation check. + """This method determines if we should apply our namespace indentation check. Args: nesting_state: The current nesting state. @@ -5888,17 +5958,17 @@ def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, only works for classes and namespaces inside of a namespace. """ - is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, - linenum) + is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, + linenum) - if not (is_namespace_indent_item or is_forward_declaration): - return False + if not (is_namespace_indent_item or is_forward_declaration): + return False - # If we are in a macro, we do not want to check the namespace indentation. - if IsMacroDefinition(raw_lines_no_comments, linenum): - return False + # If we are in a macro, we do not want to check the namespace indentation. + if IsMacroDefinition(raw_lines_no_comments, linenum): + return False - return IsBlockInNameSpace(nesting_state, is_forward_declaration) + return IsBlockInNameSpace(nesting_state, is_forward_declaration) # Call this method if the line is directly inside of a namespace. @@ -5906,16 +5976,22 @@ def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, # an inner namespace, it cannot be indented. def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum, error): - line = raw_lines_no_comments[linenum] - if Match(r'^\s+', line): - error(filename, linenum, 'runtime/indentation_namespace', 4, - 'Do not indent within a namespace') - - -def ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, + line = raw_lines_no_comments[linenum] + if Match(r'^\s+', line): + error(filename, linenum, 'runtime/indentation_namespace', 4, + 'Do not indent within a namespace') + + +def ProcessLine(filename, + file_extension, + clean_lines, + line, + include_state, + function_state, + nesting_state, + error, extra_check_functions=[]): - """Processes a single line in the file. + """Processes a single line in the file. Args: filename: Filename of the file that is being processed. @@ -5933,32 +6009,34 @@ def ProcessLine(filename, file_extension, clean_lines, line, run on each source line. Each function takes 4 arguments: filename, clean_lines, line, error """ - raw_lines = clean_lines.raw_lines - ParseNolintSuppressions(filename, raw_lines[line], line, error) - nesting_state.Update(filename, clean_lines, line, error) - CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, - error) - if nesting_state.InAsmBlock(): return - CheckForFunctionLengths(filename, clean_lines, line, function_state, error) - CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) - CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) - CheckLanguage(filename, clean_lines, line, file_extension, include_state, - nesting_state, error) - CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) - CheckForNonStandardConstructs(filename, clean_lines, line, - nesting_state, error) - CheckVlogArguments(filename, clean_lines, line, error) - CheckPosixThreading(filename, clean_lines, line, error) - CheckInvalidIncrement(filename, clean_lines, line, error) - CheckMakePairUsesDeduction(filename, clean_lines, line, error) - CheckDefaultLambdaCaptures(filename, clean_lines, line, error) - CheckRedundantVirtual(filename, clean_lines, line, error) - CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) - for check_fn in extra_check_functions: - check_fn(filename, clean_lines, line, error) + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[line], line, error) + nesting_state.Update(filename, clean_lines, line, error) + CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, + error) + if nesting_state.InAsmBlock(): return + CheckForFunctionLengths(filename, clean_lines, line, function_state, error) + CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) + CheckStyle(filename, clean_lines, line, file_extension, nesting_state, + error) + CheckLanguage(filename, clean_lines, line, file_extension, include_state, + nesting_state, error) + CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) + CheckForNonStandardConstructs(filename, clean_lines, line, nesting_state, + error) + CheckVlogArguments(filename, clean_lines, line, error) + CheckPosixThreading(filename, clean_lines, line, error) + CheckInvalidIncrement(filename, clean_lines, line, error) + CheckMakePairUsesDeduction(filename, clean_lines, line, error) + CheckDefaultLambdaCaptures(filename, clean_lines, line, error) + CheckRedundantVirtual(filename, clean_lines, line, error) + CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) + for check_fn in extra_check_functions: + check_fn(filename, clean_lines, line, error) + def FlagCxx11Features(filename, clean_lines, linenum, error): - """Flag those c++11 features that we only allow in certain places. + """Flag those c++11 features that we only allow in certain places. Args: filename: The name of the current file. @@ -5966,46 +6044,48 @@ def FlagCxx11Features(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Flag unapproved C++11 headers. - include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) - if include and include.group(1) in ('cfenv', - 'condition_variable', - 'fenv.h', - 'future', - 'mutex', - 'thread', - 'chrono', - 'ratio', - 'regex', - 'system_error', - ): - error(filename, linenum, 'build/c++11', 5, - ('<%s> is an unapproved C++11 header.') % include.group(1)) - - # The only place where we need to worry about C++11 keywords and library - # features in preprocessor directives is in macro definitions. - if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return - - # These are classes and free functions. The classes are always - # mentioned as std::*, but we only catch the free functions if - # they're not found by ADL. They're alphabetical by header. - for top_name in ( - # type_traits - 'alignment_of', - 'aligned_union', - ): - if Search(r'\bstd::%s\b' % top_name, line): - error(filename, linenum, 'build/c++11', 5, - ('std::%s is an unapproved C++11 class or function. Send c-style ' - 'an example of where it would make your code more readable, and ' - 'they may let you use it.') % top_name) - - -def ProcessFileData(filename, file_extension, lines, error, + line = clean_lines.elided[linenum] + + # Flag unapproved C++11 headers. + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + if include and include.group(1) in ( + 'cfenv', + 'condition_variable', + 'fenv.h', + 'future', + 'mutex', + 'thread', + 'chrono', + 'ratio', + 'regex', + 'system_error', ): + error(filename, linenum, 'build/c++11', 5, + ('<%s> is an unapproved C++11 header.') % include.group(1)) + + # The only place where we need to worry about C++11 keywords and library + # features in preprocessor directives is in macro definitions. + if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return + + # These are classes and free functions. The classes are always + # mentioned as std::*, but we only catch the free functions if + # they're not found by ADL. They're alphabetical by header. + for top_name in ( + # type_traits + 'alignment_of', + 'aligned_union', ): + if Search(r'\bstd::%s\b' % top_name, line): + error(filename, linenum, 'build/c++11', 5, ( + 'std::%s is an unapproved C++11 class or function. Send c-style ' + 'an example of where it would make your code more readable, and ' + 'they may let you use it.') % top_name) + + +def ProcessFileData(filename, + file_extension, + lines, + error, extra_check_functions=[]): - """Performs lint checks and reports any errors to the given error function. + """Performs lint checks and reports any errors to the given error function. Args: filename: Filename of the file that is being processed. @@ -6018,44 +6098,44 @@ def ProcessFileData(filename, file_extension, lines, error, run on each source line. Each function takes 4 arguments: filename, clean_lines, line, error """ - lines = (['// marker so line numbers and indices both start at 1'] + lines + - ['// marker so line numbers end in a known way']) + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) - include_state = _IncludeState() - function_state = _FunctionState() - nesting_state = NestingState() + include_state = _IncludeState() + function_state = _FunctionState() + nesting_state = NestingState() - ResetNolintSuppressions() + ResetNolintSuppressions() - CheckForCopyright(filename, lines, error) + CheckForCopyright(filename, lines, error) - RemoveMultiLineComments(filename, lines, error) - clean_lines = CleansedLines(lines) + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) - if file_extension == 'h': - CheckForHeaderGuard(filename, clean_lines, error) + if file_extension == 'h': + CheckForHeaderGuard(filename, clean_lines, error) - for line in xrange(clean_lines.NumLines()): - ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions) - FlagCxx11Features(filename, clean_lines, line, error) - nesting_state.CheckCompletedBlocks(filename, error) + for line in xrange(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, include_state, + function_state, nesting_state, error, extra_check_functions) + FlagCxx11Features(filename, clean_lines, line, error) + nesting_state.CheckCompletedBlocks(filename, error) - CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) - - # Check that the .cc file has included its header if it exists. - if file_extension == 'cc': - CheckHeaderFileIncluded(filename, include_state, error) + CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) - # We check here rather than inside ProcessLine so that we see raw - # lines rather than "cleaned" lines. - CheckForBadCharacters(filename, lines, error) + # Check that the .cc file has included its header if it exists. + if file_extension == 'cc': + CheckHeaderFileIncluded(filename, include_state, error) + + # We check here rather than inside ProcessLine so that we see raw + # lines rather than "cleaned" lines. + CheckForBadCharacters(filename, lines, error) + + CheckForNewlineAtEOF(filename, lines, error) - CheckForNewlineAtEOF(filename, lines, error) def ProcessConfigOverrides(filename): - """ Loads the configuration files and processes the config overrides. + """ Loads the configuration files and processes the config overrides. Args: filename: The name of the file being processed by the linter. @@ -6064,74 +6144,76 @@ def ProcessConfigOverrides(filename): False if the current |filename| should not be processed further. """ - abs_filename = os.path.abspath(filename) - cfg_filters = [] - keep_looking = True - while keep_looking: - abs_path, base_name = os.path.split(abs_filename) - if not base_name: - break # Reached the root directory. - - cfg_file = os.path.join(abs_path, "CPPLINT.cfg") - abs_filename = abs_path - if not os.path.isfile(cfg_file): - continue - - try: - with open(cfg_file) as file_handle: - for line in file_handle: - line, _, _ = line.partition('#') # Remove comments. - if not line.strip(): + abs_filename = os.path.abspath(filename) + cfg_filters = [] + keep_looking = True + while keep_looking: + abs_path, base_name = os.path.split(abs_filename) + if not base_name: + break # Reached the root directory. + + cfg_file = os.path.join(abs_path, "CPPLINT.cfg") + abs_filename = abs_path + if not os.path.isfile(cfg_file): continue - name, _, val = line.partition('=') - name = name.strip() - val = val.strip() - if name == 'set noparent': - keep_looking = False - elif name == 'filter': - cfg_filters.append(val) - elif name == 'exclude_files': - # When matching exclude_files pattern, use the base_name of - # the current file name or the directory name we are processing. - # For example, if we are checking for lint errors in /foo/bar/baz.cc - # and we found the .cfg file at /foo/CPPLINT.cfg, then the config - # file's "exclude_files" filter is meant to be checked against "bar" - # and not "baz" nor "bar/baz.cc". - if base_name: - pattern = re.compile(val) - if pattern.match(base_name): - sys.stderr.write('Ignoring "%s": file excluded by "%s". ' - 'File path component "%s" matches ' - 'pattern "%s"\n' % - (filename, cfg_file, base_name, val)) - return False - elif name == 'linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - sys.stderr.write('Line length must be numeric.') - else: + try: + with open(cfg_file) as file_handle: + for line in file_handle: + line, _, _ = line.partition('#') # Remove comments. + if not line.strip(): + continue + + name, _, val = line.partition('=') + name = name.strip() + val = val.strip() + if name == 'set noparent': + keep_looking = False + elif name == 'filter': + cfg_filters.append(val) + elif name == 'exclude_files': + # When matching exclude_files pattern, use the base_name of + # the current file name or the directory name we are processing. + # For example, if we are checking for lint errors in /foo/bar/baz.cc + # and we found the .cfg file at /foo/CPPLINT.cfg, then the config + # file's "exclude_files" filter is meant to be checked against "bar" + # and not "baz" nor "bar/baz.cc". + if base_name: + pattern = re.compile(val) + if pattern.match(base_name): + sys.stderr.write( + 'Ignoring "%s": file excluded by "%s". ' + 'File path component "%s" matches ' + 'pattern "%s"\n' % + (filename, cfg_file, base_name, val)) + return False + elif name == 'linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + sys.stderr.write('Line length must be numeric.') + else: + sys.stderr.write( + 'Invalid configuration option (%s) in file %s\n' % + (name, cfg_file)) + + except IOError: sys.stderr.write( - 'Invalid configuration option (%s) in file %s\n' % - (name, cfg_file)) - - except IOError: - sys.stderr.write( - "Skipping config file '%s': Can't open for reading\n" % cfg_file) - keep_looking = False + "Skipping config file '%s': Can't open for reading\n" % + cfg_file) + keep_looking = False - # Apply all the accumulated filters in reverse order (top-level directory - # config options having the least priority). - for filter in reversed(cfg_filters): - _AddFilters(filter) + # Apply all the accumulated filters in reverse order (top-level directory + # config options having the least priority). + for filter in reversed(cfg_filters): + _AddFilters(filter) - return True + return True def ProcessFile(filename, vlevel, extra_check_functions=[]): - """Does google-lint on a single file. + """Does google-lint on a single file. Args: filename: The name of the file to parse. @@ -6144,104 +6226,105 @@ def ProcessFile(filename, vlevel, extra_check_functions=[]): arguments: filename, clean_lines, line, error """ - _SetVerboseLevel(vlevel) - _BackupFilters() + _SetVerboseLevel(vlevel) + _BackupFilters() - if not ProcessConfigOverrides(filename): - _RestoreFilters() - return - - lf_lines = [] - crlf_lines = [] - try: - # Support the UNIX convention of using "-" for stdin. Note that - # we are not opening the file with universal newline support - # (which codecs doesn't support anyway), so the resulting lines do - # contain trailing '\r' characters if we are reading a file that - # has CRLF endings. - # If after the split a trailing '\r' is present, it is removed - # below. - if filename == '-': - lines = codecs.StreamReaderWriter(sys.stdin, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace').read().split('\n') + if not ProcessConfigOverrides(filename): + _RestoreFilters() + return + + lf_lines = [] + crlf_lines = [] + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + lines = codecs.open(filename, 'r', 'utf8', + 'replace').read().split('\n') + + # Remove trailing '\r'. + # The -1 accounts for the extra trailing blank line we get from split() + for linenum in range(len(lines) - 1): + if lines[linenum].endswith('\r'): + lines[linenum] = lines[linenum].rstrip('\r') + crlf_lines.append(linenum + 1) + else: + lf_lines.append(linenum + 1) + + except IOError: + sys.stderr.write("Skipping input '%s': Can't open for reading\n" % + filename) + _RestoreFilters() + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if filename != '-' and file_extension not in _valid_extensions: + sys.stderr.write('Ignoring %s; not a valid file name ' + '(%s)\n' % (filename, ', '.join(_valid_extensions))) else: - lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') - - # Remove trailing '\r'. - # The -1 accounts for the extra trailing blank line we get from split() - for linenum in range(len(lines) - 1): - if lines[linenum].endswith('\r'): - lines[linenum] = lines[linenum].rstrip('\r') - crlf_lines.append(linenum + 1) - else: - lf_lines.append(linenum + 1) - - except IOError: - sys.stderr.write( - "Skipping input '%s': Can't open for reading\n" % filename) - _RestoreFilters() - return - - # Note, if no dot is found, this will give the entire filename as the ext. - file_extension = filename[filename.rfind('.') + 1:] - - # When reading from stdin, the extension is unknown, so no cpplint tests - # should rely on the extension. - if filename != '-' and file_extension not in _valid_extensions: - sys.stderr.write('Ignoring %s; not a valid file name ' - '(%s)\n' % (filename, ', '.join(_valid_extensions))) - else: - ProcessFileData(filename, file_extension, lines, Error, - extra_check_functions) - - # If end-of-line sequences are a mix of LF and CR-LF, issue - # warnings on the lines with CR. - # - # Don't issue any warnings if all lines are uniformly LF or CR-LF, - # since critique can handle these just fine, and the style guide - # doesn't dictate a particular end of line sequence. - # - # We can't depend on os.linesep to determine what the desired - # end-of-line sequence should be, since that will return the - # server-side end-of-line sequence. - if lf_lines and crlf_lines: - # Warn on every line with CR. An alternative approach might be to - # check whether the file is mostly CRLF or just LF, and warn on the - # minority, we bias toward LF here since most tools prefer LF. - for linenum in crlf_lines: - Error(filename, linenum, 'whitespace/newline', 1, - 'Unexpected \\r (^M) found; better to use only \\n') + ProcessFileData(filename, file_extension, lines, Error, + extra_check_functions) - sys.stdout.write('Done processing %s\n' % filename) - _RestoreFilters() + # If end-of-line sequences are a mix of LF and CR-LF, issue + # warnings on the lines with CR. + # + # Don't issue any warnings if all lines are uniformly LF or CR-LF, + # since critique can handle these just fine, and the style guide + # doesn't dictate a particular end of line sequence. + # + # We can't depend on os.linesep to determine what the desired + # end-of-line sequence should be, since that will return the + # server-side end-of-line sequence. + if lf_lines and crlf_lines: + # Warn on every line with CR. An alternative approach might be to + # check whether the file is mostly CRLF or just LF, and warn on the + # minority, we bias toward LF here since most tools prefer LF. + for linenum in crlf_lines: + Error(filename, linenum, 'whitespace/newline', 1, + 'Unexpected \\r (^M) found; better to use only \\n') + + sys.stdout.write('Done processing %s\n' % filename) + _RestoreFilters() def PrintUsage(message): - """Prints a brief usage string and exits, optionally with an error message. + """Prints a brief usage string and exits, optionally with an error message. Args: message: The optional error message. """ - sys.stderr.write(_USAGE) - if message: - sys.exit('\nFATAL ERROR: ' + message) - else: - sys.exit(1) + sys.stderr.write(_USAGE) + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(1) def PrintCategories(): - """Prints a list of all the error-categories used by error messages. + """Prints a list of all the error-categories used by error messages. These are the categories used to filter messages via --filter. """ - sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) - sys.exit(0) + sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) + sys.exit(0) def ParseArguments(args): - """Parses the command line arguments. + """Parses the command line arguments. This may set the output format and verbosity level as side-effects. @@ -6251,82 +6334,82 @@ def ParseArguments(args): Returns: The list of filenames to lint. """ - try: - (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', - 'counting=', - 'filter=', - 'root=', - 'linelength=', - 'extensions=']) - except getopt.GetoptError: - PrintUsage('Invalid arguments.') - - verbosity = _VerboseLevel() - output_format = _OutputFormat() - filters = '' - counting_style = '' - - for (opt, val) in opts: - if opt == '--help': - PrintUsage(None) - elif opt == '--output': - if val not in ('emacs', 'vs7', 'eclipse'): - PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.') - output_format = val - elif opt == '--verbose': - verbosity = int(val) - elif opt == '--filter': - filters = val - if not filters: - PrintCategories() - elif opt == '--counting': - if val not in ('total', 'toplevel', 'detailed'): - PrintUsage('Valid counting options are total, toplevel, and detailed') - counting_style = val - elif opt == '--root': - global _root - _root = val - elif opt == '--linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - PrintUsage('Line length must be digits.') - elif opt == '--extensions': - global _valid_extensions - try: - _valid_extensions = set(val.split(',')) - except ValueError: - PrintUsage('Extensions must be comma seperated list.') - - if not filenames: - PrintUsage('No files were specified.') - - _SetOutputFormat(output_format) - _SetVerboseLevel(verbosity) - _SetFilters(filters) - _SetCountingStyle(counting_style) - - return filenames + try: + (opts, filenames) = getopt.getopt(args, '', [ + 'help', 'output=', 'verbose=', 'counting=', 'filter=', 'root=', + 'linelength=', 'extensions=' + ]) + except getopt.GetoptError: + PrintUsage('Invalid arguments.') + + verbosity = _VerboseLevel() + output_format = _OutputFormat() + filters = '' + counting_style = '' + + for (opt, val) in opts: + if opt == '--help': + PrintUsage(None) + elif opt == '--output': + if val not in ('emacs', 'vs7', 'eclipse'): + PrintUsage( + 'The only allowed output formats are emacs, vs7 and eclipse.' + ) + output_format = val + elif opt == '--verbose': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + PrintCategories() + elif opt == '--counting': + if val not in ('total', 'toplevel', 'detailed'): + PrintUsage( + 'Valid counting options are total, toplevel, and detailed') + counting_style = val + elif opt == '--root': + global _root + _root = val + elif opt == '--linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + PrintUsage('Line length must be digits.') + elif opt == '--extensions': + global _valid_extensions + try: + _valid_extensions = set(val.split(',')) + except ValueError: + PrintUsage('Extensions must be comma seperated list.') + + if not filenames: + PrintUsage('No files were specified.') + + _SetOutputFormat(output_format) + _SetVerboseLevel(verbosity) + _SetFilters(filters) + _SetCountingStyle(counting_style) + + return filenames def main(): - filenames = ParseArguments(sys.argv[1:]) + filenames = ParseArguments(sys.argv[1:]) - # Change stderr to write with replacement characters so we don't die - # if we try to print something containing non-ASCII characters. - sys.stderr = codecs.StreamReaderWriter(sys.stderr, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace') + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + sys.stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), 'replace') - _cpplint_state.ResetErrorCounts() - for filename in filenames: - ProcessFile(filename, _cpplint_state.verbose_level) - _cpplint_state.PrintErrorCounts() + _cpplint_state.ResetErrorCounts() + for filename in filenames: + ProcessFile(filename, _cpplint_state.verbose_level) + _cpplint_state.PrintErrorCounts() - sys.exit(_cpplint_state.error_count > 0) + sys.exit(_cpplint_state.error_count > 0) if __name__ == '__main__': - main() + main() diff --git a/paddle/scripts/deb/build_scripts/.gitignore b/paddle/scripts/deb/build_scripts/.gitignore new file mode 100644 index 0000000000000..1521c8b7652b1 --- /dev/null +++ b/paddle/scripts/deb/build_scripts/.gitignore @@ -0,0 +1 @@ +dist diff --git a/paddle/scripts/deb/build_scripts/Dockerfile b/paddle/scripts/deb/build_scripts/Dockerfile new file mode 100644 index 0000000000000..db365a65b7d33 --- /dev/null +++ b/paddle/scripts/deb/build_scripts/Dockerfile @@ -0,0 +1,5 @@ +FROM paddledev/paddle:gpu-latest +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +CMD cd /root/ && bash build.sh + diff --git a/paddle/scripts/deb/build_scripts/build.sh b/paddle/scripts/deb/build_scripts/build.sh new file mode 100755 index 0000000000000..d13dea514841b --- /dev/null +++ b/paddle/scripts/deb/build_scripts/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e +apt-get update +apt-get install -y dh-make +cd ~ +mkdir -p ~/dist/gpu +mkdir -p ~/dist/cpu +mkdir -p ~/dist/cpu-noavx +mkdir -p ~/dist/gpu-noavx +cd paddle +mkdir build +cd build +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/cpu + +rm -rf * +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/gpu + + +rm -rf * +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/cpu-noavx + +rm -rf * +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/gpu-noavx diff --git a/paddle/scripts/deb/build_scripts/build_deb.sh b/paddle/scripts/deb/build_scripts/build_deb.sh new file mode 100755 index 0000000000000..c38c6299f8403 --- /dev/null +++ b/paddle/scripts/deb/build_scripts/build_deb.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +docker build -t build_paddle_deb . +rm -rf dist +mkdir -p dist +docker run -v$PWD/dist:/root/dist -v $PWD/../../../..:/root/paddle --name tmp_build_deb_container build_paddle_deb +docker rm tmp_build_deb_container +docker rmi build_paddle_deb diff --git a/paddle/scripts/docker/Dockerfile.cpu b/paddle/scripts/docker/Dockerfile.cpu index 3aa8cb1a3a869..69b8363b7ac9e 100644 --- a/paddle/scripts/docker/Dockerfile.cpu +++ b/paddle/scripts/docker/Dockerfile.cpu @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=OFF ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-demo b/paddle/scripts/docker/Dockerfile.cpu-demo index 22c0b9e701bfc..ccbd183ee3c1a 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-demo +++ b/paddle/scripts/docker/Dockerfile.cpu-demo @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.cpu-devel b/paddle/scripts/docker/Dockerfile.cpu-devel index b40f3c0a30ba3..36460384f383b 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-devel +++ b/paddle/scripts/docker/Dockerfile.cpu-devel @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx b/paddle/scripts/docker/Dockerfile.cpu-noavx index 5cb5ac7dc4e68..fa3b7427b0ad3 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=OFF ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo index bec401960efb2..61315f762dee4 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel index b7c3eaed97aa5..76365311990b5 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu b/paddle/scripts/docker/Dockerfile.gpu index b7f5b6d93df50..1e023ae2818db 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/paddle/scripts/docker/Dockerfile.gpu @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=ON ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-demo b/paddle/scripts/docker/Dockerfile.gpu-demo index 2d1411de09f2a..92b0dca4026c8 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-demo +++ b/paddle/scripts/docker/Dockerfile.gpu-demo @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.gpu-devel b/paddle/scripts/docker/Dockerfile.gpu-devel index eb13f4304fa06..fb6f351fd2f7e 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-devel +++ b/paddle/scripts/docker/Dockerfile.gpu-devel @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx b/paddle/scripts/docker/Dockerfile.gpu-noavx index 0944b0e152af3..7567e62025506 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=ON ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo index 2da2a55d696a3..ac52484c5cb51 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel index 9f551462f206a..19202f306b8f7 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0a0 ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.m4 b/paddle/scripts/docker/Dockerfile.m4 index 129d21b36abd9..761aa975d6936 100644 --- a/paddle/scripts/docker/Dockerfile.m4 +++ b/paddle/scripts/docker/Dockerfile.m4 @@ -1,6 +1,7 @@ FROM PADDLE_BASE_IMAGE MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=v0.9.0 ENV WITH_GPU=PADDLE_WITH_GPU ENV IS_DEVEL=PADDLE_IS_DEVEL ENV WITH_DEMO=PADDLE_WITH_DEMO diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 33689e736cda7..ec5f3bd967d35 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -23,6 +23,7 @@ fi cd ~ git clone https://github.com/baidu/Paddle.git paddle cd paddle +git checkout ${GIT_CHECKOUT} mkdir build cd build cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ diff --git a/paddle/scripts/docker/generate.sh b/paddle/scripts/docker/generate.sh index 8a50aefd34955..2ad7527db127f 100644 --- a/paddle/scripts/docker/generate.sh +++ b/paddle/scripts/docker/generate.sh @@ -58,4 +58,3 @@ m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ -DPADDLE_WITH_AVX=OFF \ Dockerfile.m4 > Dockerfile.gpu-noavx-demo - diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index 4cf5f41f195df..20ea2fedc4d46 100644 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -28,6 +28,34 @@ function version(){ echo " with_predict_sdk: @WITH_PREDICT_SDK@" } +function ver2num() { + # convert version to number. + if [ -z "$1" ]; then # empty argument + printf "%03d%03d%03d%03d%03d" 0 + else + local VERN=$(echo $1 | sed 's#v##g' | sed 's#\.# #g' \ + | sed 's#a# 0 #g' | sed 's#b# 1 #g' | sed 's#rc# 2 #g') + if [ `echo $VERN | wc -w` -eq 3 ] ; then + printf "%03d%03d%03d%03d%03d" $VERN 999 999 + else + printf "%03d%03d%03d%03d%03d" $VERN + fi + fi +} + +PADDLE_CONF_HOME="$HOME/.config/paddle" +mkdir -p ${PADDLE_CONF_HOME} + +if [ -z "${PADDLE_NO_STAT+x}" ]; then + SERVER_VER=`curl -m 5 -X POST --data content="{ \"version\": \"@PADDLE_VERSION@\" }"\ + -b ${PADDLE_CONF_HOME}/paddle.cookie \ + -c ${PADDLE_CONF_HOME}/paddle.cookie \ + http://api.paddlepaddle.org/version 2>/dev/null` + if [ $? -eq 0 ] && [ "$(ver2num @PADDLE_VERSION@)" -lt $(ver2num $SERVER_VER) ]; then + echo "Paddle release a new version ${SERVER_VER}, you can get the install package in http://www.paddlepaddle.org" + fi +fi + MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -68,7 +96,7 @@ EOF if [ $? -eq 1 ]; then # Older version installed, or not installed at all echo "First time run paddle, need to install some python dependencies." BASEDIR=$(dirname "$0") - pip install ${BASEDIR}/../opt/paddle/share/wheels/*.whl + pip install ${BASEDIR}/../opt/paddle/share/wheels/*-@PADDLE_VERSION@-*.whl if [ $? -ne 0 ]; then echo "pip install wheels failed. " echo "Please use 'sudo paddle' at the first time you use PaddlePaddle" diff --git a/paddle/scripts/tools/build_docs/.gitignore b/paddle/scripts/tools/build_docs/.gitignore new file mode 100644 index 0000000000000..6ec14c8f5bc37 --- /dev/null +++ b/paddle/scripts/tools/build_docs/.gitignore @@ -0,0 +1,2 @@ +doc +doc_cn diff --git a/paddle/scripts/tools/build_docs/Dockerfile b/paddle/scripts/tools/build_docs/Dockerfile new file mode 100644 index 0000000000000..5db0b29c47399 --- /dev/null +++ b/paddle/scripts/tools/build_docs/Dockerfile @@ -0,0 +1,6 @@ +FROM paddledev/paddle:cpu-devel-latest +COPY build.sh / +RUN pip install sphinx &&\ + apt install -y doxygen graphviz &&\ + pip install breathe recommonmark numpy protobuf==2.6.1 +CMD /build.sh diff --git a/paddle/scripts/tools/build_docs/build.sh b/paddle/scripts/tools/build_docs/build.sh new file mode 100755 index 0000000000000..a23b6e61d4592 --- /dev/null +++ b/paddle/scripts/tools/build_docs/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -ex + +mkdir -p /build +cd /build +cmake /paddle -DWITH_DOC=ON +make paddle_docs paddle_docs_cn -j `nproc` +mkdir -p /output/doc +mkdir -p /output/doc_cn +cp -r doc/html/* /output/doc/ +cp -r doc_cn/html/* /output/doc_cn/ +cd / +rm -rf /paddle/build diff --git a/paddle/scripts/tools/build_docs/build_docs.sh b/paddle/scripts/tools/build_docs/build_docs.sh new file mode 100755 index 0000000000000..9f8b80435c8fb --- /dev/null +++ b/paddle/scripts/tools/build_docs/build_docs.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +docker build . -t paddle_build_doc +docker run --rm -v $PWD/../../../../:/paddle -v $PWD:/output paddle_build_doc diff --git a/paddle/scripts/travis/before_install.sh b/paddle/scripts/travis/before_install.linux.sh similarity index 100% rename from paddle/scripts/travis/before_install.sh rename to paddle/scripts/travis/before_install.linux.sh diff --git a/paddle/scripts/travis/before_install.osx.sh b/paddle/scripts/travis/before_install.osx.sh new file mode 100755 index 0000000000000..f438e69b822aa --- /dev/null +++ b/paddle/scripts/travis/before_install.osx.sh @@ -0,0 +1,13 @@ +#!/bin/bash +brew update +brew tap homebrew/science +brew install python +sudo pip install --upgrade protobuf==2.6.0 +brew install homebrew/versions/protobuf260 --without-python +brew install cmake python glog gflags openblas wget md5sha1sum + +wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz -O gtest.tar.gz +tar xf gtest.tar.gz +cd googletest-release-1.8.0/ +cmake . +make install diff --git a/paddle/scripts/travis/build_and_test.sh b/paddle/scripts/travis/build_and_test.sh index 3ea633be32702..242fd982aa001 100755 --- a/paddle/scripts/travis/build_and_test.sh +++ b/paddle/scripts/travis/build_and_test.sh @@ -1,7 +1,26 @@ #!/bin/bash source ./common.sh -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -make -j `nproc` -env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j `nproc`" +CMAKE_EXTRA="" +if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + CMAKE_EXTRA="-DPYTHON_LIBRARY=/usr/local/Cellar/python/2.7.12_1/Frameworks/Python.framework/Versions/2.7/lib/python2.7/config/libpython2.7.dylib" +else + CMAKE_EXTRA="-DWITH_SWIG_PY=ON" +fi + + +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA} + +NPROC=1 +if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + NRPOC=`nproc` + make -j $NPROC + make coveralls +elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + NPROC=`sysctl -n hw.ncpu` + make -j $NPROC + env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC" +fi + + sudo make install sudo paddle version diff --git a/paddle/scripts/travis/common.sh b/paddle/scripts/travis/common.sh index 37e27d665b12f..9b6e420ca7931 100755 --- a/paddle/scripts/travis/common.sh +++ b/paddle/scripts/travis/common.sh @@ -2,4 +2,3 @@ set -e mkdir -p ../../../build cd ../../../build - diff --git a/paddle/setup.py.in b/paddle/setup.py.in index 02ea9067431c6..1a15eafd5528a 100644 --- a/paddle/setup.py.in +++ b/paddle/setup.py.in @@ -18,6 +18,7 @@ from setuptools import setup, Extension import numpy as np import api.paddle_ld_flags import platform +import os system = platform.system().lower() @@ -30,8 +31,8 @@ is_lin = (system == 'linux') # because generate paddle LDFLAGS is too complicated to do in setup.py # it just read COMAKE generated LDFLAGS. extra_links = [] -ldflags = api.paddle_ld_flags.PaddleLDFlag() -ldflags = ldflags.ldflag_str() +obj = api.paddle_ld_flags.PaddleLDFlag() +ldflags = obj.ldflag_str() if ldflags is not None: extra_links.extend(ldflags.split(" ")) @@ -45,17 +46,25 @@ except: if is_lin == True: extra_links = ["-Xlinker", '-start-group'] + extra_links + ["-Xlinker", "-end-group"] elif is_osx == True: + os.environ["ARCHFLAGS"] = "-arch x86_64" extra_links = ["-Wl,-all_load"] + extra_links include_dirs = [np.get_include(), "../"] # include numpy and paddle. +extra_c = obj.c_flag() + +attr=dict() +if extra_c is not None: + attr["extra_compile_args"] = extra_c + setup(name="py_paddle", version="@PADDLE_VERSION@", ext_modules=[ Extension('py_paddle._swig_paddle', # Build SWIG Extension. ['Paddle_wrap.cxx'], include_dirs = include_dirs, - extra_link_args = extra_links + extra_link_args = extra_links, + **attr ) ], packages=['py_paddle'], diff --git a/paddle/trainer/CMakeLists.txt b/paddle/trainer/CMakeLists.txt index 08b411d2ccbae..06c019f0a9775 100644 --- a/paddle/trainer/CMakeLists.txt +++ b/paddle/trainer/CMakeLists.txt @@ -7,6 +7,7 @@ set(TRAINER_SOURCES Tester.cpp Trainer.cpp TrainerInternal.cpp + TrainerBenchmark.cpp ThreadParameterUpdater.cpp TrainerInternalConfig.cpp TrainerConfigHelper.cpp) diff --git a/paddle/trainer/ParamUtil.cpp b/paddle/trainer/ParamUtil.cpp index dae8b44b6db8e..bb309a54975a1 100644 --- a/paddle/trainer/ParamUtil.cpp +++ b/paddle/trainer/ParamUtil.cpp @@ -89,6 +89,9 @@ void ParameterUtil::saveParameters(int passId, int passInnerId) { } std::string basePath = config_->getSaveDir(); + if (basePath.find('/') == std::string::npos) { + basePath = "./" + basePath; + } mkDirRecursively(basePath.c_str()); std::string saveDir = path::join(basePath, buf); diff --git a/paddle/trainer/Tester.cpp b/paddle/trainer/Tester.cpp index ccf06e1d84edc..d3b88019faa04 100644 --- a/paddle/trainer/Tester.cpp +++ b/paddle/trainer/Tester.cpp @@ -71,24 +71,36 @@ Tester::Tester(const std::shared_ptr &config, parameterUpdater_)); } +void Tester::startTestPeriod() { + testEvaluator_->start(); + testContext_.cost = 0; + testContext_.numSamples = 0; + + parameterUpdater_->apply(); + if (intconfig_->prevBatchState) { + gradientMachine_->getState(*intconfig_->trainState); + gradientMachine_->setState(*intconfig_->testState); + } +} + +void Tester::testOneDataBatch( + const DataBatch& dataBatch, std::vector* outArgs) { + testContext_.cost += forwardOneBatch( + dataBatch, testEvaluator_.get(), outArgs); + testContext_.numSamples += dataBatch.getSize(); +} + void Tester::testOnePeriod() { DataBatch dataBatch; int64_t batchSize = config_->getOptConfig().batch_size(); - testEvaluator_->start(); - real cost = 0; - int64_t numSamples = 0; bool testAllData = intconfig_->testPeriod == 0 || intconfig_->testAllDataInOnePeriod; - int batches = testAllData ? std::numeric_limits::max() : intconfig_->testPeriod; - parameterUpdater_->apply(); - if (intconfig_->prevBatchState) { - gradientMachine_->getState(*intconfig_->trainState); - gradientMachine_->setState(*intconfig_->testState); - } + std::vector outArgs; + startTestPeriod(); for (int i = 0; i < batches; ++i) { int num = testDataProvider_->getNextBatch(batchSize, &dataBatch); if (num == 0) { @@ -102,13 +114,18 @@ void Tester::testOnePeriod() { num = testDataProvider_->getNextBatch(batchSize, &dataBatch); } } - cost += testOneBatch(dataBatch, testEvaluator_.get()); - numSamples += num; + testOneDataBatch(dataBatch, &outArgs); } + finishTestPeriod(); +} + +void Tester::finishTestPeriod() { testEvaluator_->finish(); - CHECK_GT(numSamples, 0) << "There is no samples in your test batch. Possibly " - "wrong implementation of DataProvidor.reset()"; - LOG(INFO) << " Test samples=" << numSamples << " cost=" << cost / numSamples + CHECK_GT(testContext_.numSamples, 0) + << "There is no samples in your test batch. Possibly " + "wrong implementation of DataProvidor.reset()"; + LOG(INFO) << " Test samples=" << testContext_.numSamples + << " cost=" << testContext_.cost / testContext_.numSamples << " Eval: " << *testEvaluator_; parameterUpdater_->restore(); if (intconfig_->prevBatchState) { @@ -128,9 +145,11 @@ int64_t Tester::testOneBatchById(int64_t batchId) { return 0; } + std::vector outArgs; + stats_ += std::pair{ actualBatchSize, - testOneBatch(dataBatch, testEvaluator_.get())}; + forwardOneBatch(dataBatch, testEvaluator_.get(), &outArgs)}; if (((batchId + 1) % intconfig_->logPeriod) == 0) { LOG(INFO) << " Batch=" << batchId + 1 << " " << stats_.getStats(false); @@ -139,7 +158,10 @@ int64_t Tester::testOneBatchById(int64_t batchId) { return actualBatchSize; } -real Tester::testOneBatch(const DataBatch &dataBatch, Evaluator *evaluator) { +real Tester::forwardOneBatch(const DataBatch &dataBatch, + Evaluator *evaluator, + std::vector* pOutArgs) { + auto& outArgs = *pOutArgs; const std::vector& inArgs = dataBatch.getStreams(); if (intconfig_->loadsaveParametersInPserver) { REGISTER_TIMER("prefetch"); @@ -148,12 +170,11 @@ real Tester::testOneBatch(const DataBatch &dataBatch, Evaluator *evaluator) { true /*after apply*/); } - std::vector outArgs; gradientMachine_->forward(inArgs, &outArgs, PASS_TEST); // write features if set this flag and outArgs is not empty std::string featFile = intconfig_->featFile; - if (!featFile.empty() && !outArgs.empty()) { + if (!featFile.empty() && outArgs.empty()) { size_t numOutputs = outArgs.size(); std::vector featMatrices; featMatrices.resize(numOutputs); diff --git a/paddle/trainer/Tester.h b/paddle/trainer/Tester.h index 9663b8def9145..671ffc5220eba 100644 --- a/paddle/trainer/Tester.h +++ b/paddle/trainer/Tester.h @@ -68,6 +68,10 @@ class Tester { * is training at same time. */ void testOnePeriod(); + void startTestPeriod(); + void finishTestPeriod(); + void testOneDataBatch(const DataBatch& dataBatch, + std::vector* outArgs); /** * Test for given data batch. @@ -75,7 +79,9 @@ class Tester { * @param evaluator Evaluator * @return cost */ - real testOneBatch(const DataBatch &dataBatch, Evaluator *evaluator); + real forwardOneBatch(const DataBatch& dataBatch, + Evaluator* evaluator, + std::vector* outArgs); /** @@ -99,6 +105,10 @@ class Tester { std::ofstream os_; std::vector cpuMat_; std::vector cpuVec_; + struct { + int64_t numSamples; + real cost; + } testContext_; private: /** diff --git a/paddle/trainer/ThreadParameterUpdater.cpp b/paddle/trainer/ThreadParameterUpdater.cpp index 91f7f4d29df93..d0fda1b6253e3 100644 --- a/paddle/trainer/ThreadParameterUpdater.cpp +++ b/paddle/trainer/ThreadParameterUpdater.cpp @@ -20,6 +20,8 @@ limitations under the License. */ #include "paddle/math/SparseRowMatrix.h" #include "paddle/utils/Thread.h" +P_DECLARE_int32(trainer_count); + namespace paddle { SgdThreadUpdater::SgdThreadUpdater(const OptimizationConfig& optConfig) @@ -48,6 +50,13 @@ void SgdThreadUpdater::init(std::vector& parameters) { false /*inPserver*/)); size_t numRows = para->isGradSparseUpdate() ? para->getConfig().dims(0) : 0; optimizers_[pid]->init(numRows, ¶->getConfig()); + if (para->isGradSparseUpdate() && FLAGS_trainer_count == 1) { + // For trainer_count=1, the gradient machine is NeuralNetwork, which does + // not create parameter buf for PARAMETER_GRADIENT for sparse update in + // Parameter::enableType(). But gradient parameter buf is still used + // in SgdThreadUpdater. We need to explicitly create it. + para->enableBufType(PARAMETER_GRADIENT); + } } } @@ -211,7 +220,7 @@ void SgdThreadUpdater::threadUpdateSparse( // From MultiGradientMachine SparseRowIdsCpuMatrix* mainMat = dynamic_cast( para->getMat(PARAMETER_GRADIENT).get()); - const std::vector& sparseIds = mainMat->getIds(tid); + std::vector& sparseIds = mainMat->getIds(tid); for (auto id : sparseIds) { // setup sub bufs @@ -221,6 +230,7 @@ void SgdThreadUpdater::threadUpdateSparse( optimizer->update(vecs, para->getConfig(), id); vecs[PARAMETER_GRADIENT]->zeroMem(); } + sparseIds.clear(); } else if (dynamic_cast( para->getMat(PARAMETER_GRADIENT).get())) { // From NeuralNetwork @@ -246,6 +256,10 @@ void SgdThreadUpdater::threadUpdateSparse( optimizer->update(vecs, para->getConfig(), id); vecs[PARAMETER_GRADIENT]->zeroMem(); } + // For numThreads > 1, MultiGradientMachine is used, which goes + // to the above branch. + CHECK_EQ(numThreads, 1UL); + mainMat->clearIndices(); } else { auto & m = *para->getMat(PARAMETER_GRADIENT).get(); LOG(FATAL) << "Internal error: " << para->getName() << " " diff --git a/paddle/trainer/Trainer.cpp b/paddle/trainer/Trainer.cpp index 275150e12d12b..7fc48dd1fbec6 100644 --- a/paddle/trainer/Trainer.cpp +++ b/paddle/trainer/Trainer.cpp @@ -40,7 +40,7 @@ limitations under the License. */ #include "TrainerConfigHelper.h" P_DEFINE_string(config, "", "Trainer config file"); -P_DEFINE_int32(test_period, 1000, +P_DEFINE_int32(test_period, 0, "Run test every so many train batches." " 0 for testing after each pass." " If not 0, test log_period batches." @@ -196,7 +196,8 @@ void Trainer::init(const std::shared_ptr &config, if (!dataProvider_ && config_->hasDataConfig()) { dataProvider_.reset(DataProvider::create(*config_, *config_, gpuData)); } - if (dataProvider_) { + if (!testDataProvider_) { + // No evaluator_ if there is testDataProvider but no dataProvider. evaluator_.reset(trainerInternal_.getGradientMachine()->makeEvaluator()); currentEvaluator_.reset( trainerInternal_.getGradientMachine()->makeEvaluator()); @@ -215,10 +216,7 @@ void Trainer::init(const std::shared_ptr &config, DataProvider::create(config_->getTestDataConfig(), *config_, gpuData)); } if (testDataProvider_) { - tester_.reset(new Tester(config_, createTesterConfig(), - trainerInternal_.getGradientMachine(), - trainerInternal_.getParameterUpdater(), - testDataProvider_)); + createTester(); } if (!testing && @@ -258,34 +256,25 @@ void Trainer::init(const std::shared_ptr &config, } } - // set current evaluator and evalutor trainerInternal_.setCurrentEvaluator(currentEvaluator_.get()); trainerInternal_.setEvaluator(evaluator_.get()); } void Trainer::train(size_t numPasses) { - srand(config_->getConfig().start_pass() + 1); - dataProvider_->reset(); - - if (this->testDataProvider_) { - this->testDataProvider_->reset(); - } - - trainerInternal_.getGradientMachine()->start(*config_, dataProvider_); - + startTrain(); for (size_t i = 0; i < numPasses; ++i) { if (IGradientMachineMode::trainWholeDataInOneBatch(mode_)) { trainOnePassBatch(config_->getConfig().start_pass() + i); } else { - trainOnePass(config_->getConfig().start_pass() + i); + trainOnePass(); } if (i < numPasses - 1) { dataProvider_->reset(); } } - trainerInternal_.getGradientMachine()->finish(); + finishTrain(); } @@ -387,13 +376,30 @@ real Trainer::checkGradient() { return maxDiff; } -void Trainer::trainOnePass(int passId) { - this->stats_->reset(); - int64_t batchId = 0; - int32_t batchSize = config_->getOptConfig().batch_size(); - real avgTestCost = 0; - int64_t numAvgTests = 0; - int passInnerId = 1; +void Trainer::startTrain() { + trainPassContext_.passId = config_->getConfig().start_pass(); + srand(config_->getConfig().start_pass() + 1); + if (dataProvider_) { + dataProvider_->reset(); + } + + if (this->testDataProvider_) { + this->testDataProvider_->reset(); + } + + trainerInternal_.getGradientMachine()->start(*config_, dataProvider_); +} + +void Trainer::finishTrain() { + trainerInternal_.getGradientMachine()->finish(); +} + +void Trainer::startTrainPass() { + stats_->reset(); + trainPassContext_.batchId = 0; + trainPassContext_.avgTestCost = 0; + trainPassContext_.numAvgTests = 0; + trainPassContext_.passInnerId = 1; trainerInternal_.getParameterUpdater()->startPass(); evaluator_->start(); @@ -401,81 +407,83 @@ void Trainer::trainOnePass(int passId) { trainerInternal_.getGradientMachine()->resetState(); trainerInternal_.getGradientMachine()->getState(testState_); } - while (true) { - DataBatch dataBatch; - - int num = 0; - { - REGISTER_TIMER("getTrainBatch"); - num = dataProvider_->getNextBatch(batchSize, &dataBatch); - } - if (num == 0) break; +} - if (averageEvaluator_) { - int64_t mod = batchId % FLAGS_average_test_period; - if (mod >= FLAGS_average_test_period - FLAGS_log_period) { - if (mod == FLAGS_average_test_period - FLAGS_log_period) { - averageEvaluator_->start(); - } - trainerInternal_.getParameterUpdater()->apply(); - if (FLAGS_prev_batch_state) { - trainerInternal_.getGradientMachine()->getState(trainState_); - } - avgTestCost += - tester_->testOneBatch(dataBatch, averageEvaluator_.get()); - if (FLAGS_prev_batch_state) { - trainerInternal_.getGradientMachine()->setState(trainState_); - } - numAvgTests += num; - trainerInternal_.getParameterUpdater()->restore(); +void Trainer::trainOneDataBatch(DataBatch& dataBatch) { + int num = dataBatch.getSize(); + if (averageEvaluator_) { + int64_t mod = trainPassContext_.batchId % FLAGS_average_test_period; + if (mod >= FLAGS_average_test_period - FLAGS_log_period) { + if (mod == FLAGS_average_test_period - FLAGS_log_period) { + averageEvaluator_->start(); } + trainerInternal_.getParameterUpdater()->apply(); + if (FLAGS_prev_batch_state) { + trainerInternal_.getGradientMachine()->getState(trainState_); + } + trainPassContext_.avgTestCost += + tester_->forwardOneBatch( + dataBatch, averageEvaluator_.get(), &forwardOutput_); + if (FLAGS_prev_batch_state) { + trainerInternal_.getGradientMachine()->setState(trainState_); + } + trainPassContext_.numAvgTests += num; + trainerInternal_.getParameterUpdater()->restore(); } - { - REGISTER_TIMER("TrainBatch"); - trainerInternal_.trainOneBatch(batchId, dataBatch); - } + } + { + REGISTER_TIMER("TrainBatch"); + trainerInternal_.trainOneBatch( + trainPassContext_.batchId, dataBatch, &forwardOutput_); + } - if (averageEvaluator_ && - batchId % FLAGS_average_test_period == FLAGS_average_test_period - 1) { - averageEvaluator_->finish(); - LOG(INFO) << " Averaged parameter:" - << " cost=" << avgTestCost / numAvgTests - << " Eval: " << *averageEvaluator_; - numAvgTests = 0; - avgTestCost = 0; - } + if (averageEvaluator_ && + trainPassContext_.batchId % FLAGS_average_test_period + == FLAGS_average_test_period - 1) { + averageEvaluator_->finish(); + LOG(INFO) << " Averaged parameter:" + << " cost=" << trainPassContext_.avgTestCost + / trainPassContext_.numAvgTests + << " Eval: " << *averageEvaluator_; + trainPassContext_.numAvgTests = 0; + trainPassContext_.avgTestCost = 0; + } - ++batchId; + ++trainPassContext_.batchId; - if (batchId % FLAGS_log_period == 0) { - FOR_TIMING(globalStat.setThreadInfo(true)); - FOR_TIMING(globalStat.printAllStatus()); - FOR_TIMING(globalStat.reset()); - } + if (trainPassContext_.batchId % FLAGS_log_period == 0) { + FOR_TIMING(globalStat.setThreadInfo(true)); + FOR_TIMING(globalStat.printAllStatus()); + FOR_TIMING(globalStat.reset()); + } - if (testDataProvider_ && FLAGS_test_period > 0 && - batchId % FLAGS_test_period == 0) { - tester_->testOnePeriod(); - } + if (testDataProvider_ && FLAGS_test_period > 0 && + trainPassContext_.batchId % FLAGS_test_period == 0) { + tester_->testOnePeriod(); + } - if (FLAGS_saving_period_by_batches > 0 && - batchId > FLAGS_saving_period_by_batches * passInnerId && - 0 == FLAGS_trainer_id) { - trainerInternal_.getParameterUpdater()->catchUpWith(); - if (testDataProvider_) { - tester_->testOnePeriod(); - } - paramUtil_->saveParametersOnePass(passId, passInnerId); - ++passInnerId; + if (FLAGS_saving_period_by_batches > 0 && + trainPassContext_.batchId + > FLAGS_saving_period_by_batches * trainPassContext_.passInnerId && + 0 == FLAGS_trainer_id) { + trainerInternal_.getParameterUpdater()->catchUpWith(); + if (testDataProvider_) { + tester_->testOnePeriod(); } + paramUtil_->saveParametersOnePass( + trainPassContext_.passId, trainPassContext_.passInnerId); + ++trainPassContext_.passInnerId; } +} - if (batchId == 0) { +void Trainer::finishTrainPass() { + if (trainPassContext_.batchId == 0) { // This means no more data from DataProvider return; } - trainerInternal_.finishTrainPass(passId, batchId); + trainerInternal_.finishTrainPass( + trainPassContext_.passId, trainPassContext_.batchId); FOR_TIMING(globalStat.setThreadInfo(true)); FOR_TIMING(globalStat.printAllStatus()); @@ -485,9 +493,30 @@ void Trainer::trainOnePass(int passId) { tester_->testOnePeriod(); } - if (passId % FLAGS_saving_period == 0 && FLAGS_trainer_id == 0) { - paramUtil_->saveParametersOnePass(passId); + if (trainPassContext_.passId % FLAGS_saving_period == 0 + && FLAGS_trainer_id == 0) { + paramUtil_->saveParametersOnePass(trainPassContext_.passId); } + ++trainPassContext_.passId; +} + +void Trainer::trainOnePass() { + startTrainPass(); + size_t batchSize = config_->getOptConfig().batch_size(); + while (true) { + DataBatch dataBatch; + + int num = 0; + { + REGISTER_TIMER("getTrainBatch"); + num = dataProvider_->getNextBatch(batchSize, &dataBatch); + } + if (num == 0) break; + CHECK_EQ(num, dataBatch.getSize()); + trainOneDataBatch(dataBatch); + } + + finishTrainPass(); } void Trainer::trainOnePassBatch(int passId) { @@ -582,6 +611,13 @@ void Trainer::clearGradient() { int Trainer::getBatchSize() { return config_->getOptConfig().batch_size(); } +void Trainer::createTester() { + tester_.reset(new paddle::Tester(config_, createTesterConfig(), + trainerInternal_.getGradientMachine(), + trainerInternal_.getParameterUpdater(), + testDataProvider_)); +} + void Trainer::test() { tester_->test(); } diff --git a/paddle/trainer/Trainer.h b/paddle/trainer/Trainer.h index 9bfd6d107a204..7762722456c44 100644 --- a/paddle/trainer/Trainer.h +++ b/paddle/trainer/Trainer.h @@ -94,6 +94,12 @@ class Trainer { */ real checkGradient(); + void startTrain(); + void finishTrain(); + void startTrainPass(); + void finishTrainPass(); + void trainOneDataBatch(DataBatch& dataBatch); + void time(); /** * given a dataBatch and the current parameter value @@ -144,11 +150,11 @@ class Trainer { protected: /** - * Train one pass of data. passId starts from 0 + * Train one pass of data. * * SGD Method. */ - void trainOnePass(int passId); + void trainOnePass(); /** * Train one pass in one batch. @@ -161,6 +167,8 @@ class Trainer { */ void clearGradient(); + void createTester(); + private: std::unique_ptr createTesterConfig(); @@ -173,6 +181,17 @@ class Trainer { MachineState trainState_; MachineState testState_; + struct TrainPassContext { + int64_t batchId; + real avgTestCost; + int64_t numAvgTests; + int passId; + int passInnerId; + }; + std::vector forwardOutput_; + + TrainPassContext trainPassContext_; + std::unique_ptr evaluator_; std::unique_ptr currentEvaluator_; std::unique_ptr averageEvaluator_; diff --git a/paddle/trainer/TrainerBenchmark.cpp b/paddle/trainer/TrainerBenchmark.cpp new file mode 100644 index 0000000000000..54862e95b4a73 --- /dev/null +++ b/paddle/trainer/TrainerBenchmark.cpp @@ -0,0 +1,71 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#undef PADDLE_DISABLE_TIMER + +#include "Trainer.h" +#include "paddle/utils/Stat.h" +#include "paddle/utils/Util.h" + +P_DECLARE_int32(test_period); + +P_DEFINE_bool(feed_data, false, "Wether to read data from DataProvider."); + +namespace paddle { + +void Trainer::time() { + startTrain(); + + trainerInternal_.getParameterUpdater()->startPass(); + evaluator_->start(); + + DataBatch dataBatch; + int32_t batchSize = config_->getOptConfig().batch_size(); + int32_t num = dataProvider_->getNextBatch(batchSize, &dataBatch); + CHECK_EQ(num, batchSize) << "The sample number is less than batch size " + << num << " != " << batchSize; + + CHECK(dataBatch.getSize()) << "No data from data provider"; + + std::vector outputs; + // burning time + LOG(INFO) << "Burning time..."; + for (int n = 0; n < 10; ++n) { + trainerInternal_.trainOneBatch(n, dataBatch, &outputs); + } + LOG(INFO) << "Burning time end."; + + for (int n = 0; n < FLAGS_test_period; n++) { + if (FLAGS_feed_data) { + REGISTER_TIMER("GetData"); + num = dataProvider_->getNextBatch(batchSize, &dataBatch); + } + + if (num != batchSize) { + break; + } + + { + REGISTER_TIMER("FwdBwd"); + trainerInternal_.trainOneBatch(n, dataBatch, &outputs); + } + } + globalStat.setThreadInfo(true); + globalStat.printSegTimerStatus(); + globalStat.reset(); + + finishTrain(); +} + +} // namespace paddle diff --git a/paddle/trainer/TrainerInternal.cpp b/paddle/trainer/TrainerInternal.cpp index 6029a4b2c1d0a..e23e42927c381 100644 --- a/paddle/trainer/TrainerInternal.cpp +++ b/paddle/trainer/TrainerInternal.cpp @@ -55,6 +55,8 @@ void TrainerInternal::init(const std::shared_ptr &config, gradientMachine_ = gradientMachine; if (!gradientMachine) { + CHECK(config_->getConfig().has_model_config()) + << "Missing model_config in trainer_config"; gradientMachine_.reset(GradientMachine::create( config_->getConfig().model_config(), intconfig_->mode, parameterUpdater_->getParameterTypes())); @@ -62,7 +64,8 @@ void TrainerInternal::init(const std::shared_ptr &config, } void TrainerInternal::trainOneBatch(int64_t batchId, - const DataBatch& dataBatch) { + const DataBatch& dataBatch, + std::vector* outArgs) { // true means updating parameter whenever gradient is ready during backward() bool doPipelineUpdate = (intconfig_->mode != GradientMachine::kSgdSparseCpuTraining) && @@ -84,7 +87,6 @@ void TrainerInternal::trainOneBatch(int64_t batchId, } const std::vector& inArgs = dataBatch.getStreams(); - std::vector outArgs; PassType passType = parameterUpdater_->startBatch(actualBatchSize); @@ -114,7 +116,7 @@ void TrainerInternal::trainOneBatch(int64_t batchId, timer.start(); #endif REGISTER_TIMER("forwardBackward"); - forwardBackwardBatch(inArgs, outArgs, passType, updateCallback, + forwardBackwardBatch(inArgs, *outArgs, passType, updateCallback, doPipelineUpdate); #ifndef PADDLE_DISABLE_TIMER timer.stop(); @@ -132,7 +134,7 @@ void TrainerInternal::trainOneBatch(int64_t batchId, real cost = 0; { REGISTER_TIMER("sumCost"); - cost = Argument::sumCosts(outArgs); + cost = Argument::sumCosts(*outArgs); } if (batchId % intconfig_->log_period == 0) { diff --git a/paddle/trainer/TrainerInternal.h b/paddle/trainer/TrainerInternal.h index 17011c4d2e46f..3a53aa1d17b31 100644 --- a/paddle/trainer/TrainerInternal.h +++ b/paddle/trainer/TrainerInternal.h @@ -81,7 +81,9 @@ class TrainerInternal { * @param batchId current batch id * @param dataBatch data for the batch */ - void trainOneBatch(int64_t batchId, const DataBatch& dataBatch); + void trainOneBatch(int64_t batchId, + const DataBatch& dataBatch, + std::vector* outArgs); /** * showParameterStats diff --git a/paddle/trainer/TrainerMain.cpp b/paddle/trainer/TrainerMain.cpp index 94266639f94ad..a486cc383ace6 100644 --- a/paddle/trainer/TrainerMain.cpp +++ b/paddle/trainer/TrainerMain.cpp @@ -103,6 +103,8 @@ int main(int argc, char** argv) { trainer.checkGradient(); } else if (FLAGS_job == "test") { trainer.test(); + } else if (FLAGS_job == "time") { + trainer.time(); } else { LOG(FATAL) << "Unknown job type: " << FLAGS_job; } diff --git a/paddle/trainer/tests/__init__.py b/paddle/trainer/tests/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/paddle/trainer/tests/__init__.py +++ b/paddle/trainer/tests/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/trainer/tests/config_parser_test.py b/paddle/trainer/tests/config_parser_test.py index 5ca874cec7914..c5ec315d6b01b 100644 --- a/paddle/trainer/tests/config_parser_test.py +++ b/paddle/trainer/tests/config_parser_test.py @@ -17,6 +17,6 @@ if __name__ == '__main__': parse_config_and_serialize('trainer/tests/test_config.conf', '') parse_config_and_serialize( - 'trainer/tests/sample_trainer_config.conf', + 'trainer/tests/sample_trainer_config.conf', 'extension_module_name=paddle.trainer.config_parser_extension') parse_config_and_serialize('gserver/tests/pyDataProvider/trainer.conf', '') diff --git a/paddle/trainer/tests/gen_proto_data.py b/paddle/trainer/tests/gen_proto_data.py index c818a94bee7c2..a3dbc10c886e1 100644 --- a/paddle/trainer/tests/gen_proto_data.py +++ b/paddle/trainer/tests/gen_proto_data.py @@ -21,8 +21,7 @@ import pprint logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) @@ -36,33 +35,32 @@ # [[-1,0], [0,0]] means previous token at column 0 and current token at # column 0 are combined as one feature. patterns = [ - [[-2,0]], - [[-1,0]], - [[0,0]], - [[1,0]], - [[2,0]], - - [[-1,0], [0,0]], - [[0,0], [1,0]], - - [[-2,1]], - [[-1,1]], - [[0,1]], - [[1,1]], - [[2,1]], - [[-2,1], [-1,1]], - [[-1,1], [0,1]], - [[0,1], [1,1]], - [[1,1], [2,1]], - - [[-2,1], [-1,1], [0,1]], - [[-1,1], [0,1], [1,1]], - [[0,1], [1,1], [2,1]], + [[-2, 0]], + [[-1, 0]], + [[0, 0]], + [[1, 0]], + [[2, 0]], + [[-1, 0], [0, 0]], + [[0, 0], [1, 0]], + [[-2, 1]], + [[-1, 1]], + [[0, 1]], + [[1, 1]], + [[2, 1]], + [[-2, 1], [-1, 1]], + [[-1, 1], [0, 1]], + [[0, 1], [1, 1]], + [[1, 1], [2, 1]], + [[-2, 1], [-1, 1], [0, 1]], + [[-1, 1], [0, 1], [1, 1]], + [[0, 1], [1, 1], [2, 1]], ] + def make_features(sequence): length = len(sequence) num_features = len(sequence[0]) + def get_features(pos): if pos < 0: return ['#B%s' % -pos] * num_features @@ -72,9 +70,10 @@ def get_features(pos): for i in xrange(length): for pattern in patterns: - fname = '/'.join([get_features(i+pos)[f] for pos, f in pattern]) + fname = '/'.join([get_features(i + pos)[f] for pos, f in pattern]) sequence[i].append(fname) + ''' Source file format: Each line is for one timestep. The features are separated by space. @@ -87,6 +86,8 @@ def get_features(pos): return a list of dict for each column ''' + + def create_dictionaries(filename, cutoff, oov_policy): def add_to_dict(sequence, dicts): num_features = len(dicts) @@ -118,7 +119,6 @@ def add_to_dict(sequence, dicts): features = line.split(' ') sequence.append(features) - for i in xrange(num_features): dct = dicts[i] n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 @@ -161,12 +161,9 @@ def write_proto(file, message): if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist in dicts[i]. ''' -def gen_proto_file( - input_file, - dicts, - oov_policy, - output_file): + +def gen_proto_file(input_file, dicts, oov_policy, output_file): def write_sequence(out, sequence): num_features = len(dicts) is_beginning = True @@ -213,8 +210,8 @@ def write_sequence(out, sequence): if patterns: slot_def = header.slot_defs.add() slot_def.type = DataFormat.SlotDef.VECTOR_SPARSE_NON_VALUE - slot_def.dim = sum([len(dicts[i]) - for i in xrange(num_original_columns, len(dicts))]) + slot_def.dim = sum( + [len(dicts[i]) for i in xrange(num_original_columns, len(dicts))]) logger.info("feature_dim=%s" % slot_def.dim) for i in xrange(num_original_columns): @@ -242,30 +239,31 @@ def write_sequence(out, sequence): logger.info("num_sequences=%s" % num_sequences) + dict2 = { - 'B-ADJP': 0, - 'I-ADJP': 1, - 'B-ADVP': 2, - 'I-ADVP': 3, - 'B-CONJP': 4, - 'I-CONJP': 5, - 'B-INTJ': 6, - 'I-INTJ': 7, - 'B-LST': 8, - 'I-LST': 9, - 'B-NP': 10, - 'I-NP': 11, - 'B-PP': 12, - 'I-PP': 13, - 'B-PRT': 14, - 'I-PRT': 15, - 'B-SBAR': 16, - 'I-SBAR': 17, - 'B-UCP': 18, - 'I-UCP': 19, - 'B-VP': 20, - 'I-VP': 21, - 'O': 22 + 'B-ADJP': 0, + 'I-ADJP': 1, + 'B-ADVP': 2, + 'I-ADVP': 3, + 'B-CONJP': 4, + 'I-CONJP': 5, + 'B-INTJ': 6, + 'I-INTJ': 7, + 'B-LST': 8, + 'I-LST': 9, + 'B-NP': 10, + 'I-NP': 11, + 'B-PP': 12, + 'I-PP': 13, + 'B-PRT': 14, + 'I-PRT': 15, + 'B-SBAR': 16, + 'I-SBAR': 17, + 'B-UCP': 18, + 'I-UCP': 19, + 'B-VP': 20, + 'I-VP': 21, + 'O': 22 } if __name__ == '__main__': @@ -273,16 +271,9 @@ def write_sequence(out, sequence): cutoff += [3] * len(patterns) oov_policy = [OOV_POLICY_IGNORE, OOV_POLICY_ERROR, OOV_POLICY_ERROR] oov_policy += [OOV_POLICY_IGNORE] * len(patterns) - dicts = create_dictionaries( - 'trainer/tests/train.txt', cutoff, oov_policy) + dicts = create_dictionaries('trainer/tests/train.txt', cutoff, oov_policy) dicts[2] = dict2 - gen_proto_file( - 'trainer/tests/train.txt', - dicts, - oov_policy, - 'trainer/tests/train_proto.bin') - gen_proto_file( - 'trainer/tests/test.txt', - dicts, - oov_policy, - 'trainer/tests/test_proto.bin') + gen_proto_file('trainer/tests/train.txt', dicts, oov_policy, + 'trainer/tests/train_proto.bin') + gen_proto_file('trainer/tests/test.txt', dicts, oov_policy, + 'trainer/tests/test_proto.bin') diff --git a/paddle/trainer/tests/test.txt b/paddle/trainer/tests/test.txt index 68e7f72e3d8d4..3ad503b34f2e1 100644 --- a/paddle/trainer/tests/test.txt +++ b/paddle/trainer/tests/test.txt @@ -998,4 +998,3 @@ from IN B-PP Friday NNP B-NP 's POS B-NP Tokyo NNP I-NP - diff --git a/paddle/trainer/tests/testPyDataWrapper.py b/paddle/trainer/tests/testPyDataWrapper.py index 49bd760f4e20e..4607bec24e1fe 100644 --- a/paddle/trainer/tests/testPyDataWrapper.py +++ b/paddle/trainer/tests/testPyDataWrapper.py @@ -21,7 +21,10 @@ import string -@provider(slots=[SparseNonValueSlot(10), DenseSlot(2), SparseValueSlot(10), StringSlot(1), IndexSlot(3)]) +@provider(slots=[ + SparseNonValueSlot(10), DenseSlot(2), SparseValueSlot(10), StringSlot(1), + IndexSlot(3) +]) def processNonSequenceData(obj, filename): with open(filename, "rb") as f: for line in f: @@ -50,6 +53,7 @@ def __values_mapper__(s): seq_count_randomer = lambda: random.randrange(1, SEQUENCE_LIMIT) str_count_randomer = lambda: random.randrange(1, STRING_LIMIT) + class IDRandomer(): # A random generator, return unique id def __init__(self): self.id_set = set() @@ -61,38 +65,57 @@ def __call__(self): return idx else: return self.__call__() + + # SparseValueSlot def sparse_value_creator(_): rand = IDRandomer() return [(rand(), val_randomer()) for _ in xrange(sparse_count_randomer())] + + sparse_value = map(sparse_value_creator, range(seq_count_randomer())) + # DenseSlot def dense_creator(_): return [val_randomer() for _ in xrange(SPARSE_ID_LIMIT)] + + dense = map(dense_creator, range(seq_count_randomer())) + # SparseNonValueSlot def sparse_creator(_): rand = IDRandomer() return [rand() for _ in xrange(sparse_count_randomer())] + + sparse_nonvalue = map(sparse_creator, range(seq_count_randomer())) # IndexSlot ids = [sparse_id_randomer() for _ in range(seq_count_randomer())] + # StringSlot -def random_str(size = 8, chars=string.ascii_letters + string.digits): +def random_str(size=8, chars=string.ascii_letters + string.digits): return ''.join(random.choice(chars) for _ in range(size)) + + strs = [random_str(str_count_randomer()) for _ in range(seq_count_randomer())] + def processSeqAndGenerateDataInit(obj, *args, **kwargs): obj.json_filename = kwargs.get("load_data_args", "test_data.json") -@provider(slots=[SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), - SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), - StringSlot(SPARSE_ID_LIMIT)], - use_seq=True, init_hook=processSeqAndGenerateDataInit) + +@provider( + slots=[ + SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), + SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), + StringSlot(SPARSE_ID_LIMIT) + ], + use_seq=True, + init_hook=processSeqAndGenerateDataInit) def processSeqAndGenerateData(obj, name): retv = [sparse_value, dense, sparse_nonvalue, ids, strs] # Write to protoseq. @@ -104,10 +127,15 @@ def processSeqAndGenerateData(obj, name): def processSubSeqAndGenerateDataInit(obj, *args, **kwargs): obj.json_filename = kwargs.get("load_data_args", "test_data.json") -@provider(slots=[SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), - SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), - StringSlot(SPARSE_ID_LIMIT)], - use_seq=True, init_hook=processSubSeqAndGenerateDataInit) + +@provider( + slots=[ + SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), + SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), + StringSlot(SPARSE_ID_LIMIT) + ], + use_seq=True, + init_hook=processSubSeqAndGenerateDataInit) def processSubSeqAndGenerateData(obj, name): retv_json = [sparse_value, dense, sparse_nonvalue, ids, strs] retv_wrapper = [[sparse_value], [dense], [sparse_nonvalue], [ids], [strs]] @@ -116,6 +144,7 @@ def processSubSeqAndGenerateData(obj, name): json.dump(retv_json, f) yield retv_wrapper + if __name__ == "__main__": pvd = processNonSequenceData("test.txt") print pvd.getNextBatch(100) diff --git a/paddle/trainer/tests/test_CompareSparse.cpp b/paddle/trainer/tests/test_CompareSparse.cpp index ff37d7b364840..311dd333a1b16 100644 --- a/paddle/trainer/tests/test_CompareSparse.cpp +++ b/paddle/trainer/tests/test_CompareSparse.cpp @@ -57,7 +57,7 @@ std::vector trainerOnePassTest(const string& configFile, << " sparseUpdate=" << sparseUpdate; srand(FLAGS_seed); *ThreadLocalRand::getSeed() = FLAGS_seed; - + ThreadLocalRandomEngine::get().seed(FLAGS_seed); if (useGpu) { CHECK_LE(trainerCount, gNumDevices); } diff --git a/paddle/trainer/tests/test_config.conf b/paddle/trainer/tests/test_config.conf index 5d2e2ba9df5c7..664e18cb98681 100644 --- a/paddle/trainer/tests/test_config.conf +++ b/paddle/trainer/tests/test_config.conf @@ -13,157 +13,71 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Todo(luotao02) This config is only used for unitest. It is out of date now, and will be updated later. - -default_initial_std(0.5) - -model_type("nn") - -DataLayer( - name = "input", - size = 3, -) - -DataLayer( - name = "weight", - size = 1, -) - -Layer( - name = "layer1_1", - type = "fc", - size = 5, - active_type = "sigmoid", - inputs = "input", -) - -Layer( - name = "layer1_2", - type = "fc", - size = 12, - active_type = "linear", - inputs = Input("input", parameter_name='sharew'), -) - -Layer( - name = "layer1_3", - type = "fc", - size = 3, - active_type = "tanh", - inputs = "input", -) - -Layer( - name = "layer1_5", - type = "fc", - size = 3, - active_type = "tanh", - inputs = Input("input", - learning_rate=0.01, - momentum=0.9, - decay_rate=0.05, - initial_mean=0.0, - initial_std=0.01, - format = "csc", - nnz = 4) -) - -FCLayer( - name = "layer1_4", - size = 5, - active_type = "square", - inputs = "input", - drop_rate = 0.5, -) - -Layer( - name = "pool", - type = "pool", - inputs = Input("layer1_2", - pool = Pool(pool_type="cudnn-avg-pool", - channels = 1, - size_x = 2, - size_y = 3, - img_width = 3, - padding = 1, - padding_y = 2, - stride = 2, - stride_y = 3)) -) - -Layer( - name = "concat", - type = "concat", - inputs = ["layer1_3", "layer1_4"], -) - -MixedLayer( - name = "output", - size = 3, - active_type = "softmax", - inputs = [ - FullMatrixProjection("layer1_1", - learning_rate=0.1), - TransposedFullMatrixProjection("layer1_2", parameter_name='sharew'), - FullMatrixProjection("concat"), - IdentityProjection("layer1_3"), - ], -) - -Layer( - name = "label", - type = "data", - size = 1, -) - -Layer( - name = "cost", - type = "multi-class-cross-entropy", - inputs = ["output", "label", "weight"], -) - -Layer( - name = "cost2", - type = "nce", - num_classes = 3, - active_type = "sigmoid", - neg_sampling_dist = [0.1, 0.3, 0.6], - inputs = ["layer1_2", "label", "weight"], -) - -Evaluator( - name = "error", - type = "classification_error", - inputs = ["output", "label", "weight"] -) - -Inputs("input", "label", "weight") -Outputs("cost", "cost2") - -TrainData( - ProtoData( - files = "dummy_list", - constant_slots = [1.0], - async_load_data = True, - ) -) - -TestData( - SimpleData( - files = "trainer/tests/sample_filelist.txt", - feat_dim = 3, - context_len = 0, - buffer_capacity = 1000000, - async_load_data = False, - ), -) - -Settings( - algorithm = "sgd", - num_batches_per_send_parameter = 1, - num_batches_per_get_parameter = 1, - batch_size = 100, - learning_rate = 0.001, - learning_rate_decay_a = 1e-5, - learning_rate_decay_b = 0.5, -) +from paddle.trainer_config_helpers import * + +TrainData(ProtoData( + files = "dummy_list", + constant_slots = [1.0], + async_load_data = True)) + +TestData(SimpleData( + files = "trainer/tests/sample_filelist.txt", + feat_dim = 3, + context_len = 0, + buffer_capacity = 1000000, + async_load_data = False)) + +settings(batch_size = 100) + +data = data_layer(name='input', size=3) + +wt = data_layer(name='weight', size=1) + +fc1 = fc_layer(input=data, size=5, + bias_attr=True, + act=SigmoidActivation()) + +fc2 = fc_layer(input=data, size=12, + bias_attr=True, + param_attr=ParamAttr(name='sharew'), + act=LinearActivation()) + +fc3 = fc_layer(input=data, size=3, + bias_attr=True, + act=TanhActivation()) + +fc4 = fc_layer(input=data, size=5, + bias_attr=True, + layer_attr=ExtraAttr(drop_rate=0.5), + act=SquareActivation()) + +pool = img_pool_layer(input=fc2, + pool_size=2, + pool_size_y=3, + num_channels=1, + padding=1, + padding_y=2, + stride=2, + stride_y=3, + img_width=3, + pool_type=CudnnAvgPooling()) + +concat = concat_layer(input=[fc3, fc4]) + +with mixed_layer(size=3, act=SoftmaxActivation()) as output: + output += full_matrix_projection(input=fc1) + output += trans_full_matrix_projection(input=fc2, + param_attr=ParamAttr(name='sharew')) + output += full_matrix_projection(input=concat) + output += identity_projection(input=fc3) + +lbl = data_layer(name='label', size=1) + +cost = classification_cost(input=output, label=lbl, weight=wt, + layer_attr=ExtraAttr(device=-1)) + +nce = nce_layer(input=fc2, label=lbl, weight=wt, + num_classes=3, + neg_distribution=[0.1, 0.3, 0.6]) + +outputs(cost, nce) diff --git a/paddle/trainer/tests/test_gen_dict.txt b/paddle/trainer/tests/test_gen_dict.txt index 91a84146180e0..1000f90057824 100644 --- a/paddle/trainer/tests/test_gen_dict.txt +++ b/paddle/trainer/tests/test_gen_dict.txt @@ -6,4 +6,4 @@ 5 6 7 -8 \ No newline at end of file +8 diff --git a/paddle/trainer/tests/train.txt b/paddle/trainer/tests/train.txt index 8d9b15dcf5bba..2313aee987ba7 100644 --- a/paddle/trainer/tests/train.txt +++ b/paddle/trainer/tests/train.txt @@ -4998,4 +4998,3 @@ However RB B-ADVP the DT B-NP disclosure NN I-NP of IN B-PP - diff --git a/paddle/utils/.gitignore b/paddle/utils/.gitignore new file mode 100644 index 0000000000000..f2cfd7409412d --- /dev/null +++ b/paddle/utils/.gitignore @@ -0,0 +1 @@ +enable_virtualenv.c diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index 0557b01e36f07..45240b5002aa1 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -2,6 +2,9 @@ file(GLOB UTIL_HEADERS . *.h) file(GLOB UTIL_SOURCES . *.cpp) +create_resources(enable_virtualenv.py enable_virtualenv.c) +set(UTIL_RES enable_virtualenv.c) + if(APPLE) file(GLOB UTIL_ARCH_SOURCES . arch/osx/*.cpp) else() @@ -9,7 +12,8 @@ else() endif() add_library(paddle_utils STATIC ${UTIL_SOURCES} - ${UTIL_ARCH_SOURCES}) + ${UTIL_ARCH_SOURCES} + ${UTIL_RES}) add_style_check_target(paddle_utils ${UTIL_HEADERS}) add_style_check_target(paddle_utils ${UTIL_SOURCES} ${UTIL_ARCH_SOURCES}) diff --git a/paddle/utils/Logging.h b/paddle/utils/Logging.h index b3f439804686f..7fdfa3240c1de 100644 --- a/paddle/utils/Logging.h +++ b/paddle/utils/Logging.h @@ -191,7 +191,7 @@ void installFailureWriter(void(*callback)(const char*, int)); } #endif // PADDLE_USE_GLOG -#ifdef NDEBUG +#ifndef NDEBUG #define DEBUG_LEVEL 5 #define DBG VLOG(DEBUG_LEVEL) #else diff --git a/paddle/utils/PythonUtil.cpp b/paddle/utils/PythonUtil.cpp index 78c3a80674f9c..90e5093f96ea4 100644 --- a/paddle/utils/PythonUtil.cpp +++ b/paddle/utils/PythonUtil.cpp @@ -77,11 +77,18 @@ static std::recursive_mutex g_pyMutex; PyGuard::PyGuard() : guard_(g_pyMutex) {} -static void printPyErrorStack(std::ostream& os, bool withEndl = false) { +static void printPyErrorStack(std::ostream& os, bool withEndl = false, + bool withPyPath = true) { PyObject * ptype, *pvalue, *ptraceback; PyErr_Fetch(&ptype, &pvalue, &ptraceback); PyErr_NormalizeException(&ptype, &pvalue, &ptraceback); PyErr_Clear(); + if (withPyPath) { + os << "Current PYTHONPATH: " << py::repr(PySys_GetObject(strdup("path"))); + if (withEndl) { + os << std::endl; + } + } PyTracebackObject* obj = (PyTracebackObject*)ptraceback; os << "Python Error: " << PyString_AsString(PyObject_Str(ptype)) @@ -114,10 +121,7 @@ PyObjectPtr callPythonFuncRetPyObj(const std::string& moduleName, const std::string& funcName, const std::vector& args) { PyGuard guard; - PyObjectPtr pyModuleName(PyString_FromString(moduleName.c_str())); - CHECK_PY(pyModuleName) << "Import PyModule failed" << moduleName; - PyObjectPtr pyModule(PyImport_Import(pyModuleName.get())); - CHECK_PY(pyModule) << "Import Python Module"<< moduleName << " failed."; + PyObjectPtr pyModule = py::import(moduleName); PyObjectPtr pyFunc(PyObject_GetAttrString(pyModule.get(), funcName.c_str())); CHECK_PY(pyFunc) << "GetAttrString failed."; PyObjectPtr pyArgs(PyTuple_New(args.size())); @@ -143,7 +147,7 @@ PyObjectPtr createPythonClass( const std::vector& args, const std::map& kwargs) { PyGuard guard; - PyObjectPtr pyModule(PyImport_ImportModule(moduleName.c_str())); + PyObjectPtr pyModule = py::import(moduleName); LOG(INFO) << "createPythonClass moduleName.c_str:" << moduleName.c_str(); CHECK_PY(pyModule) << "Import module " << moduleName << " failed."; PyObjectPtr pyDict(PyModule_GetDict(pyModule.get())); @@ -181,18 +185,29 @@ std::string getPyCallStack() { printPyErrorStack(os, true); return os.str(); } + +PyObjectPtr import(const std::string &moduleName) { + auto module = PyImport_ImportModule(moduleName.c_str()); + CHECK_PY(module) << "Import " << moduleName << "Error"; + return PyObjectPtr(module); +} + } // namespace py #endif - +extern "C" { +extern const char enable_virtualenv_py[]; +} void initPython(int argc, char** argv) { #ifndef PADDLE_NO_PYTHON Py_SetProgramName(argv[0]); Py_Initialize(); PySys_SetArgv(argc, argv); - // python blocks SIGINT. Need to enable it. signal(SIGINT, SIG_DFL); + + // Manually activate virtualenv when user is using virtualenv + PyRun_SimpleString(enable_virtualenv_py); #endif } diff --git a/paddle/utils/PythonUtil.h b/paddle/utils/PythonUtil.h index db02d1252b405..00fc177022ac3 100644 --- a/paddle/utils/PythonUtil.h +++ b/paddle/utils/PythonUtil.h @@ -87,6 +87,8 @@ PyObjectPtr createPythonClass(const std::string& moduleName, CHECK((x) != nullptr) << ::paddle::py::getPyCallStack() namespace py { +PyObjectPtr import(const std::string& moduleName); + /** * Cast a PyLong or PyInt to int type T. * @tparam T return type. diff --git a/paddle/utils/Queue.h b/paddle/utils/Queue.h index d73f27d7fafd6..f952cf58778de 100644 --- a/paddle/utils/Queue.h +++ b/paddle/utils/Queue.h @@ -135,6 +135,21 @@ class Queue { queueCV_.wait(lock, [this]() { return numElements_ == 0; }); } + /** + * @brief wait queue is not empty at most for some seconds. + * @param seconds wait time limit. + * @return true if queue is not empty. false if timeout. + */ + bool waitNotEmptyFor(int seconds) { + std::unique_lock lock(queueLock_); + return queueCV_.wait_for( + lock, + std::chrono::seconds(seconds), + [this] { + return numElements_ != 0; + }); + } + private: std::deque elements_; int numElements_; diff --git a/paddle/utils/ThreadLocal.h b/paddle/utils/ThreadLocal.h index 686a1a99a4aa0..b91e4ad5472ca 100644 --- a/paddle/utils/ThreadLocal.h +++ b/paddle/utils/ThreadLocal.h @@ -22,6 +22,7 @@ limitations under the License. */ #include #include #include +#include "Util.h" #include "Logging.h" namespace paddle { @@ -156,14 +157,7 @@ class ThreadLocalD { static void dataDestructor(void* p) { delete (T*)p; } void updateMap(T* p) { -#if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); -#else - #ifndef __NR_gettid - #define __NR_gettid 224 - #endif - pid_t tid = syscall(__NR_gettid); -#endif + pid_t tid = getTID(); CHECK_NE(tid, -1); std::lock_guard guard(mutex_); auto ret = threadMap_.insert(std::make_pair(tid, p)); diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index c3c76f907d40e..b16d4314654ff 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -95,14 +95,18 @@ namespace paddle { pid_t getTID() { #if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); + // syscall is deprecated: first deprecated in macOS 10.12. + // syscall is unsupported; + // syscall pid_t tid = syscall(SYS_thread_selfid); + uint64_t tid; + pthread_threadid_np(NULL, &tid); #else #ifndef __NR_gettid #define __NR_gettid 224 #endif pid_t tid = syscall(__NR_gettid); #endif - CHECK_NE(tid, -1); + CHECK_NE((int)tid, -1); return tid; } @@ -374,7 +378,7 @@ hl_activation_mode_t hlActiveType(const std::string& type) { return HL_ACTIVATION_RELU; } else if (type == "tanh") { return HL_ACTIVATION_TANH; - } else if (type == "linear") { + } else if (type == "linear" || type == "") { return HL_ACTIVATION_LINEAR; } else { LOG(FATAL) << "Do not support activation type " << type; diff --git a/paddle/utils/arch/osx/Locks.cpp b/paddle/utils/arch/osx/Locks.cpp index 47e44e9d7c114..b3ec454976520 100644 --- a/paddle/utils/arch/osx/Locks.cpp +++ b/paddle/utils/arch/osx/Locks.cpp @@ -15,7 +15,9 @@ limitations under the License. */ #include "paddle/utils/Locks.h" #include "paddle/utils/Logging.h" #include +#include #include + namespace paddle { class SemaphorePrivate { @@ -50,21 +52,19 @@ void Semaphore::post() { class SpinLockPrivate { public: - SpinLockPrivate(): lock_(OS_SPINLOCK_INIT) {} - - OSSpinLock lock_; - char padding_[64 - sizeof(OSSpinLock)]; // Padding to cache line size + std::atomic_flag lock_ = ATOMIC_FLAG_INIT; + char padding_[64 - sizeof(lock_)]; // Padding to cache line size }; SpinLock::SpinLock(): m(new SpinLockPrivate()) {} SpinLock::~SpinLock() { delete m; } void SpinLock::lock() { - OSSpinLockLock(&m->lock_); + while (m->lock_.test_and_set(std::memory_order_acquire)) {} } void SpinLock::unlock() { - OSSpinLockUnlock(&m->lock_); + m->lock_.clear(std::memory_order_release); } diff --git a/paddle/utils/enable_virtualenv.py b/paddle/utils/enable_virtualenv.py new file mode 100644 index 0000000000000..ccfaa7c147b2c --- /dev/null +++ b/paddle/utils/enable_virtualenv.py @@ -0,0 +1,12 @@ +import os + + +def __activate_virtual_env__(): + __path__ = os.getenv('VIRTUAL_ENV') + if __path__ is None: + return + __script__ = os.path.join(__path__, 'bin', 'activate_this.py') + execfile(__script__, {'__file__': __script__}) + + +__activate_virtual_env__() diff --git a/paddle/utils/tests/CMakeLists.txt b/paddle/utils/tests/CMakeLists.txt index 51f1889392845..adf489fafe722 100644 --- a/paddle/utils/tests/CMakeLists.txt +++ b/paddle/utils/tests/CMakeLists.txt @@ -4,6 +4,7 @@ add_simple_unittest(test_Thread) add_simple_unittest(test_StringUtils) add_simple_unittest(test_CustomStackTrace) add_simple_unittest(test_ThreadBarrier) +add_simple_unittest(test_SpinLock) add_executable( test_CustomStackTracePrint diff --git a/paddle/utils/tests/test_CommandLineParser.cpp b/paddle/utils/tests/test_CommandLineParser.cpp index d5f6018864cb9..9bb6827540f61 100644 --- a/paddle/utils/tests/test_CommandLineParser.cpp +++ b/paddle/utils/tests/test_CommandLineParser.cpp @@ -109,4 +109,3 @@ int main(int argc, char** argv) { } #endif - diff --git a/paddle/utils/tests/test_SpinLock.cpp b/paddle/utils/tests/test_SpinLock.cpp new file mode 100644 index 0000000000000..ebc84e0f52d82 --- /dev/null +++ b/paddle/utils/tests/test_SpinLock.cpp @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include "paddle/utils/Logging.h" +#include "paddle/utils/CommandLineParser.h" +#include "paddle/utils/Util.h" +#include "paddle/utils/Locks.h" + +P_DEFINE_int32(test_thread_num, 100, "testing thread number"); + +void testNormalImpl(size_t thread_num, const std::function + & callback) { + paddle::SpinLock mutex; + std::vector threads; + threads.reserve(thread_num); + + size_t count = 0; + for (size_t i = 0; i < thread_num; ++i) { + threads.emplace_back([&thread_num, &count, &mutex, &callback]{ + callback(thread_num, count, mutex); + }); + } + for (auto& thread : threads) { + thread.join(); + } + // Check whether all threads reach this point or not + CHECK_EQ(count, thread_num); +} + +TEST(ThreadSpinLock, normalTest) { + for (auto &thread_num : {10, 30, 50 , 100 , 300, 1000}) { + testNormalImpl(thread_num, [](size_t thread_num, + size_t& count, paddle::SpinLock& mutex){ + std::lock_guard lock(mutex); + ++count; + }); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + paddle::initMain(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index b32f8b1ee9072..aea77248cbac0 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -88,7 +88,8 @@ message PoolConfig { required uint32 size_x = 3; // Tell the net where in the input image to start the pooling. - required uint32 start = 4; + // start is deprecated now. + optional uint32 start = 4; // Defines the stride size between successive pooling squares. required uint32 stride = 5; @@ -119,6 +120,14 @@ message PoolConfig { optional uint32 padding_y = 13 [default = 0]; } +message SppConfig { + required string pool_type = 1; + required uint32 pyramid_height = 2; + required uint32 channels = 3; + required uint32 img_size = 4; + optional uint32 img_size_y = 5; +} + message NormConfig { // rnorm or cmrnorm required string norm_type = 1; @@ -169,6 +178,15 @@ message BlockExpandConfig { required uint32 img_size_y = 11; } +message MaxOutConfig { + required uint32 channels = 1; + required uint32 groups = 2; + + // The size of input feature map. + required uint32 img_size_x = 3; + required uint32 img_size_y = 4; +} + message ProjectionConfig { required string type = 1; required string name = 2; @@ -186,6 +204,9 @@ message ProjectionConfig { // For IdentityOffsetProjection optional uint64 offset = 11 [default = 0]; + + // For pool + optional PoolConfig pool_conf = 12; } message OperatorConfig { @@ -202,6 +223,15 @@ message OperatorConfig { optional int32 num_filters = 7; } +message BilinearInterpConfig { + // The size of input feature map. + optional uint32 img_size_x = 1; + optional uint32 img_size_y = 2; + // The size of output feature map. + required uint32 out_size_x = 3; + required uint32 out_size_y = 4; + required uint32 num_channels = 5; +} message ImageConfig { // The image data dimensionality. @@ -224,6 +254,9 @@ message LayerInputConfig { // If the input layer has multi-output. // Set the argument name. optional string input_layer_argument = 9; + optional BilinearInterpConfig bilinear_interp_conf = 10; + optional MaxOutConfig maxout_conf = 11; + optional SppConfig spp_conf = 12; } message LayerConfig { @@ -244,7 +277,7 @@ sinclude(`ModelConfigLayer.proto.m4') // (which is how convnets are usually trained). Setting this to // false will untie the biases, yielding a separate bias for // every location at which the filter is applied. - optional bool shared_biases = 8; + optional bool shared_biases = 8 [default = false]; // Valid values are ones that divide the area of the output // grid in this convolutional layer. For example if this layer @@ -368,6 +401,18 @@ sinclude(`ModelConfigLayer.proto.m4') // use to compute moving mean and variance. optional real moving_average_fraction = 47 [default = 0.9]; + + // bias size + optional uint32 bias_size = 48 [default = 0]; + + // this parameter can be used as a user-defined parameter when necessary, + // without changing the proto file. + // e.g., when a new layer with a user-defined parameter is implemented, + // it can be used to pass that parameter, without modifying the proto file. + // string type is used for flexibility: different types can be converted + // to string and reinterpreted in the user's own layer implementation. + optional string user_arg = 49; + } message EvaluatorConfig { diff --git a/proto/TrainerConfig.proto.m4 b/proto/TrainerConfig.proto.m4 index a42ff88d54b5e..3b0e24f90bed8 100644 --- a/proto/TrainerConfig.proto.m4 +++ b/proto/TrainerConfig.proto.m4 @@ -130,7 +130,7 @@ message OptimizationConfig { }; message TrainerConfig { - required ModelConfig model_config = 1; + optional ModelConfig model_config = 1; optional DataConfig data_config = 2; required OptimizationConfig opt_config = 3; optional DataConfig test_data_config = 4; diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/proto/__init__.py b/python/paddle/proto/__init__.py index 7f9e87eee6037..cd6a59ecbb095 100644 --- a/python/paddle/proto/__init__.py +++ b/python/paddle/proto/__init__.py @@ -12,3 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. +from paddle.proto.TrainerConfig_pb2 import OptimizationConfig, TrainerConfig +from paddle.proto.ModelConfig_pb2 import ModelConfig diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 34f5dd41b7e68..0c577ec657bc6 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -18,9 +18,8 @@ import functools import itertools -logging.basicConfig( - format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") +logging.basicConfig(format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" + " %(message)s") class SequenceType(object): @@ -132,8 +131,10 @@ def __init__(self, generator, input_order): def __call__(self, obj, filename): for item in self.generator(obj, filename): if isinstance(item, dict): - yield [item.get(input_name, None) for input_name in - self.input_order] + yield [ + item.get(input_name, None) + for input_name in self.input_order + ] else: yield item @@ -162,8 +163,8 @@ def __call__(self, obj, filename): yield items except AssertionError as e: self.logger.warning( - "Item (%s) is not fit the input type with error %s" - % (repr(item), repr(e))) + "Item (%s) is not fit the input type with error %s" % + (repr(item), repr(e))) if self.check_fail_continue: continue @@ -202,14 +203,17 @@ def loop_check(callback, item): callback(each) -def provider(input_types=None, should_shuffle=None, pool_size=-1, +def provider(input_types=None, + should_shuffle=None, + pool_size=-1, min_pool_size=-1, can_over_batch_size=True, calc_batch_size=None, cache=CacheType.NO_CACHE, - check=False, check_fail_continue=False, - use_dynamic_order=True, - init_hook=None, **kwargs): + check=False, + check_fail_continue=False, + init_hook=None, + **kwargs): """ Provider decorator. Use it to make a function into PyDataProvider2 object. In this function, user only need to get each sample for some train/test @@ -228,9 +232,15 @@ def process(settings, file_name): The configuration of data provider should be setup by\: :param input_types: Specify the input types, can also be set in init_hook. - It is a list of InputType object. For example, input_types= \ - [dense_vector(9), integer_value(2)]. - :type input_types: list|tuple + It could be a list of InputType object. For example, + input_types=[dense_vector(9), integer_value(2)]. Or user + can set a dict of InputType object, which key is + data_layer's name. For example, input_types=\ + {'img': img_features, 'label': label}. when using dict of + InputType, user could yield a dict of feature values, which + key is also data_layer's name. + + :type input_types: list|tuple|dict :param should_shuffle: True if data should shuffle. Pass None means shuffle when is training and not to shuffle when is testing. @@ -281,12 +291,6 @@ def process(settings, file_name): drop the wrong format data when it is True. Has no effect when check set to False. :type check_fail_continue: bool - - :param use_dynamic_order: Allow provider to yield a dictionary object, whose - key is a input data layer name, and value is the - feature value. The tuples are still allowed when - use_dynmaic_order is True. - :type use_dynamic_order: bool """ def __wrapper__(generator): @@ -319,9 +323,9 @@ def __init__(self, file_list, **kwargs): "Could not recognize should_shuffle (%s), " "just use default value of should_shuffle." " Please set should_shuffle to bool value or " - "something in %s" % ( - repr(self.should_shuffle), - repr(true_table + false_table))) + "something in %s" % + (repr(self.should_shuffle), + repr(true_table + false_table))) self.should_shuffle = None self.pool_size = pool_size @@ -340,6 +344,11 @@ def __init__(self, file_list, **kwargs): assert self.slots is not None assert self.generator is not None + use_dynamic_order = False + if isinstance(self.slots, dict): # reorder input_types + self.slots = [self.slots[ipt] for ipt in self.input_order] + use_dynamic_order = True + if len(self.slots) == 1: self.generator = SingleSlotWrapper(self.generator) @@ -347,8 +356,7 @@ def __init__(self, file_list, **kwargs): self.generator = InputOrderWrapper(self.generator, self.input_order) if self.check: - self.generator = CheckWrapper(self.generator, - self.slots, + self.generator = CheckWrapper(self.generator, self.slots, check_fail_continue, self.logger) @@ -364,4 +372,3 @@ def deserialize_args(args): :return: """ return cPickle.loads(args) - diff --git a/python/paddle/trainer/PyDataProviderWrapper.py b/python/paddle/trainer/PyDataProviderWrapper.py index c4b907af54699..90b684a000017 100644 --- a/python/paddle/trainer/PyDataProviderWrapper.py +++ b/python/paddle/trainer/PyDataProviderWrapper.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ This module provide a wrapper(decorator) to wrap a data process method into a PyDataProvider. Some examples are shown `here `_. @@ -47,6 +46,7 @@ import io + class SlotType(object): # Just a hint for user. pass @@ -83,6 +83,7 @@ class SparseNonValueSlot(SlotType): - **SubSeq**: [[[int, int, ...], [int, ....], ...] , \ [[int, int, ...], [int, ....], ...] , ...] """ + def __init__(self, dim): """ :param dim: slot dimension @@ -294,8 +295,9 @@ def reset(self): fn = "%s_%d" % (self.profile_filename, self.profile_count) sortby = "cumulative" with open(fn, "w") as f: - pstats.Stats(self.profiler, stream=f).sort_stats( - sortby).print_stats() + pstats.Stats( + self.profiler, + stream=f).sort_stats(sortby).print_stats() self.logger.info("saving profile to file %s" % fn) self.profile_count += 1 self.logger.info("resetting profile") @@ -453,9 +455,10 @@ def writeDataStream(dat, data_callback): seq_stream.flush() subseq_stream.flush() - return "".join([self.int_packer.pack(current_batch_size), - data_bytes.getvalue(), - seq_bytes.getvalue(), subseq_bytes.getvalue()]) + return "".join([ + self.int_packer.pack(current_batch_size), data_bytes.getvalue(), + seq_bytes.getvalue(), subseq_bytes.getvalue() + ]) finally: data_stream.close() @@ -516,7 +519,7 @@ def __prepareData(self, batch_size, ret_list): self.data_pool[idx]) idx -= 1 - ret_list += self.data_pool[self.data_pool_idx: idx + 1] + ret_list += self.data_pool[self.data_pool_idx:idx + 1] # for speed reason, just shift left index, not delete data actually. self.data_pool_idx = idx + 1 @@ -537,8 +540,8 @@ def fillPool(self): if self.max_pool_size == 0: for i in xrange(min(self.file_count, len(self.generators))): self.data_pool += list(self.generators[i]) - self.generators = self.generators[ - min(self.file_count, len(self.generators)):] + self.generators = self.generators[min(self.file_count, + len(self.generators)):] self.max_pool_size = len(self.data_pool) else: while len(self.data_pool) < self.max_pool_size and len( @@ -562,9 +565,15 @@ def default_init_hook(cls, *args, **kwargs): del cls, args, kwargs -def provider(slots=None, use_seq=False, should_shuffle=True, pool_size=1, - can_over_batch_size=True, calc_batch_size=lambda data: 1, - debug=False, init_hook=default_init_hook, profile_filename=None): +def provider(slots=None, + use_seq=False, + should_shuffle=True, + pool_size=1, + can_over_batch_size=True, + calc_batch_size=lambda data: 1, + debug=False, + init_hook=default_init_hook, + profile_filename=None): """ The decorator for PyDataProvider. User should use this to create Provider class. User should only concern how to read sample from file. @@ -663,7 +672,7 @@ class Cls(GeneralPyDataProvider): def __init__(self, *file_list, **kwargs): logging.basicConfig( format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") + " %(message)s") self.logger = logging.getLogger("") if debug: diff --git a/python/paddle/trainer/__init__.py b/python/paddle/trainer/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/python/paddle/trainer/__init__.py +++ b/python/paddle/trainer/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 1f55298f24f07..dbe2f3b29278c 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -13,7 +13,6 @@ # limitations under the License. from __future__ import print_function - ''' The following functions are available in the config file: @@ -101,50 +100,45 @@ raise logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) __real_print__ = print -print=logger.info +print = logger.info # from layer type name to layer class g_layer_type_map = {} + # Initialize global variables. We use this function so that we can # call parse_config() multiple times def init_config_environment( - g_default_momentum = None, - g_default_decay_rate = None, - g_default_initial_mean = 0., - g_default_initial_std = 0.01, - g_default_num_batches_regularization = None, - g_default_initial_strategy = 0, - g_default_initial_smart = False, - g_default_gradient_clipping_threshold = None, - g_default_device = None, - g_default_update_hooks = None, - g_default_compact_func = None, - - g_config = TrainerConfig(), - g_layer_map = {}, - g_parameter_map = {}, - - g_extended_config_funcs = {}, + g_default_momentum=None, + g_default_decay_rate=None, + g_default_initial_mean=0., + g_default_initial_std=0.01, + g_default_num_batches_regularization=None, + g_default_initial_strategy=0, + g_default_initial_smart=False, + g_default_gradient_clipping_threshold=None, + g_default_device=None, + g_default_update_hooks=None, + g_default_compact_func=None, + g_config=TrainerConfig(), + g_layer_map={}, + g_parameter_map={}, + g_extended_config_funcs={}, # store command args of paddle_trainer - g_command_config_args = {}, + g_command_config_args={}, # Used for PyDataProvider to avoid duplicate module name - g_py_module_name_list = [], - - g_current_submodel = None, - g_root_submodel = None, - g_submodel_map = {}, - g_submodel_stack = [], - - g_add_submodel_suffix = False, - ): + g_py_module_name_list=[], + g_current_submodel=None, + g_root_submodel=None, + g_submodel_map={}, + g_submodel_stack=[], + g_add_submodel_suffix=False, ): for k, v in locals().iteritems(): globals()[k] = copy.deepcopy(v) @@ -161,43 +155,54 @@ def config_assert(b, msg): if not b: logger.fatal(msg) + g_config_funcs = {} + # decorator for indicating a function which can be used in config file def config_func(func): g_config_funcs[func.func_name] = func return func + # decorator for indicating a class which can be used in config file def config_class(cls): g_config_funcs[cls.__name__] = cls return cls + # decorator for indicating a class for a layer type def config_layer(layer_type): def wrap(cls): g_config_funcs[cls.__name__] = cls g_layer_type_map[layer_type] = cls return cls + return wrap + def gen_parameter_name(layer_name, input_index): return '_%s.w%d' % (layer_name, input_index) + def gen_bias_parameter_name(layer_name): return '_%s.wbias' % layer_name + def default(x, default_value): return default_value if x is None else x + class Cfg(object): def add_keys(self, locals): for k, v in locals.iteritems(): if not k.startswith('_'): self.__setattr__(k, v) + # functions available in config file + # Define the name of the input layers of the NeuralNetwork. # The type of these layers must be "data". # These layers will be provided with the DataBatch obtained @@ -217,6 +222,11 @@ def Inputs(*args): g_config.model_config.input_layer_names.append(name) +@config_func +def HasInputsSet(): + return len(g_current_submodel.input_layer_names) != 0 + + # Define the name of the output layers of the NeuralNetwork. # Usually the output is simply the cost layer. # You can specify other layers as outputs and calculate the @@ -240,7 +250,7 @@ def SubModelBegin(name): global g_current_submodel, g_root_submodel, g_submodel_stack g_submodel_stack.append(g_current_submodel) - name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel + name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel config_assert(name not in g_submodel_map, 'Duplicated submodel name: %s' % name) @@ -250,36 +260,42 @@ def SubModelBegin(name): g_submodel_map[name] = sub_model g_current_submodel = sub_model + @config_func -def SubModelEnd(name = None): +def SubModelEnd(name=None): global g_current_submodel, g_root_submodel, g_submodel_stack - config_assert(g_current_submodel is not g_root_submodel, "submodel not begin") + config_assert(g_current_submodel is not g_root_submodel, + "submodel not begin") if name is not None: - config_assert(g_current_submodel.name == MakeLayerNameInParentSubmodel(name), - "submodel name error") + config_assert( + g_current_submodel.name == MakeLayerNameInParentSubmodel(name), + "submodel name error") g_current_submodel = g_submodel_stack.pop() + def MakeLayerNameInParentSubmodel(name): suffix = "" if len(g_submodel_stack) > 1: suffix = "@" + g_submodel_stack[-1].name return name + suffix + def GetLayerBaseName(name): return name.split('@')[0] -def MakeLayerNameInSubmodel(name, submodel_name = None): + +def MakeLayerNameInSubmodel(name, submodel_name=None): global g_current_submodel global g_add_submodel_suffix - if (submodel_name is None - and not g_add_submodel_suffix - and not g_current_submodel.is_recurrent_layer_group): + if (submodel_name is None and not g_add_submodel_suffix and + not g_current_submodel.is_recurrent_layer_group): return name if submodel_name is None: submodel_name = g_current_submodel.name return name + "@" + submodel_name + # Define a recurrent layer group begin with RecurrentLayerGroupBegin # and end with RecurrentLayerGroupEnd. # A recurrent layer group forward/backward one frame after previous frame @@ -328,8 +344,10 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, if in_links_count == 0: in_links_has_subseq = has_subseq else: - config_assert(in_links_has_subseq == has_subseq, - "The sequence type of in_links should be the same in RecurrentLayerGroup") + config_assert( + in_links_has_subseq == has_subseq, + "The sequence type of in_links should be the same in RecurrentLayerGroup" + ) in_links_count += 1 layer_name = MakeLayerNameInParentSubmodel(name) layer = g_layer_map[layer_name] @@ -343,6 +361,7 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, pair.link_name = MakeLayerNameInSubmodel(name) pair.has_subseq = has_subseq + @config_func def RecurrentLayerGroupSetOutLink(link): if isinstance(link, basestring): @@ -359,8 +378,7 @@ def RecurrentLayerGroupSetOutLink(link): def RecurrentLayerGroupSetGenerator(generator=None): - generator.eos_layer_name = MakeLayerNameInSubmodel( - generator.eos_layer_name) + generator.eos_layer_name = MakeLayerNameInSubmodel(generator.eos_layer_name) g_current_submodel.generator.CopyFrom(generator) @@ -371,21 +389,18 @@ def RecurrentLayerGroupBegin(name, generator=None, target_inlinkname="", seq_reversed=False): - RecurrentLayerGroupWithoutOutLinksBegin(name, - in_links, - seq_reversed, + RecurrentLayerGroupWithoutOutLinksBegin(name, in_links, seq_reversed, target_inlinkname) for link in out_links: RecurrentLayerGroupSetOutLink(link) - if generator is not None: RecurrentLayerGroupSetGenerator(generator) - config_assert(len(in_links) == 0, - "no in_links should be passed to generator") - config_assert(len(out_links) >= 1, - "one or more than one out_links should be passed to generator") - + config_assert( + len(in_links) == 0, "no in_links should be passed to generator") + config_assert( + len(out_links) >= 1, + "one or more than one out_links should be passed to generator") @config_func @@ -393,9 +408,10 @@ def RecurrentLayerGroupEnd(name): global g_current_submodel config_assert(g_current_submodel.is_recurrent_layer_group, "RecurrentLayerGroup not begin") - for pair in g_current_submodel.memories: #check exist + for pair in g_current_submodel.memories: #check exist layer = g_layer_map[pair.layer_name] - config_assert(layer is not None, "memory declare wrong name:%s" % pair.layer_name) + config_assert(layer is not None, + "memory declare wrong name:%s" % pair.layer_name) memory_link = g_layer_map[pair.link_name] config_assert(layer.size == memory_link.size, "memory declare wrong size:%d" % memory_link.size) @@ -414,12 +430,14 @@ def RecurrentLayerGroupEnd(name): else: GatherAgentLayer(name=agent_name, size=layer.size) + # Define the model type # currently, the paddle supports "nn", "recurrent_nn", "recursive_nn" and "multi_nn" @config_func def model_type(name): g_config.model_config.type = name + @config_class class Bias(Cfg): def __init__( @@ -437,10 +455,10 @@ def __init__( sparse_remote_update=None, gradient_clipping_threshold=None, is_static=None, - is_shared=None, - ): + is_shared=None, ): self.add_keys(locals()) + # Define one input for a layer @config_class class Input(Cfg): @@ -461,28 +479,32 @@ def __init__( sparse_update=None, gradient_clipping_threshold=None, conv=None, + bilinear_interp=None, norm=None, pool=None, image=None, block_expand=None, + maxout=None, + spp=None, format=None, nnz=None, is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) + # Define a projection for iexed layer @config_class class Projection(Input): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, input_layer_name, - size = 0, # projection output size + size=0, # projection output size parameter_name=None, learning_rate=None, momentum=None, @@ -502,8 +524,7 @@ def __init__( is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) @@ -517,8 +538,10 @@ def __init__( # to indicate using the size from Layer config def calc_output_size(self, input_layer_config): return self.size + def calc_parameter_size(self, input_size, output_size): raise NotimplementedError + def calc_parameter_dims(self, input_size, output_size): raise NotimplementedError @@ -529,31 +552,32 @@ class IdentityProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # Like IdentityProjection, but layer size may smaller than input size, # the projection select dimesions [offset, offset+layer_size) from input @config_class class IdentityOffsetProjection(Projection): type = 'identity_offset' - def __init__( - self, - input_layer_name, - offset, - **xargs): - super(IdentityOffsetProjection, self).__init__( - input_layer_name, **xargs) + def __init__(self, input_layer_name, offset, **xargs): + super(IdentityOffsetProjection, self).__init__(input_layer_name, + **xargs) self.proj_conf.offset = offset def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # DotMulProjection performs element-wise multiplication with weight @config_class class DotMulProjection(Projection): @@ -561,49 +585,67 @@ class DotMulProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return output_size + def calc_parameter_dims(self, input_size, output_size): return [1, output_size] +# ScalingProjection +@config_class +class ScalingProjection(Projection): + type = 'scaling' + + def calc_output_size(self, input_layer_config): + return input_layer_config.size + + def calc_parameter_size(self, input_size, output_size): + return 1 + + def calc_parameter_dims(self, input_size, output_size): + return [1, 1] + + @config_class class TableProjection(Projection): type = 'table' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class FullMatrixProjection(Projection): type = 'fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class TransposedFullMatrixProjection(Projection): type = 'trans_fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [output_size, input_size] + @config_class class ContextProjection(Projection): type = 'context' - def __init__( - self, - input_layer_name, - context_start, - context_length, - trainable_padding, - **xargs): + def __init__(self, input_layer_name, context_start, context_length, + trainable_padding, **xargs): super(ContextProjection, self).__init__(input_layer_name, **xargs) self.proj_conf.context_start = context_start self.proj_conf.context_length = context_length @@ -627,14 +669,51 @@ def calc_parameter_dims(self, input_size, output_size): _total_pad = 0 +@config_class +class ConvProjection(Projection): + type = 'conv' + + def __init__(self, + input_layer_name, + num_filters=None, + conv_conf=None, + **xargs): + super(ConvProjection, self).__init__(input_layer_name, **xargs) + + if num_filters is not None: + self.proj_conf.num_filters = num_filters + + parse_conv(conv_conf, input_layer_name, self.proj_conf.conv_conf, + num_filters) + # TODO: support rectangle input + self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x + **2) * num_filters + + def calc_output_size(self, input_layer_config): + return self.proj_conf.output_size + + def calc_parameter_size(self, input_size, output_size): + co = self.proj_conf.num_filters + ci = self.proj_conf.conv_conf.channels + fh = self.proj_conf.conv_conf.filter_size + fw = self.proj_conf.conv_conf.filter_size_y + return co * ci * fh * fw + + def calc_bias_size(self): + return self.proj_conf.num_filters + + def calc_parameter_dims(self, input_size, output_size): + return None + + # Define a operator for mixed layer @config_class class Operator(Cfg): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, - input_layer_names, - ): + input_layer_names, ): self.add_keys(locals()) self.operator_conf = OperatorConfig() self.operator_conf.type = self.type @@ -645,16 +724,13 @@ def check_dims(self): def calc_output_size(self, input_sizes): return 0 + @config_class class DotMulOperator(Operator): type = 'dot_mul' - def __init__( - self, - input_layer_names, - scale=None, - **xargs): - super(DotMulOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, input_layer_names, scale=None, **xargs): + super(DotMulOperator, self).__init__(input_layer_names, **xargs) if scale is not None: self.operator_conf.dotmul_scale = scale @@ -670,25 +746,24 @@ def calc_output_size(self, input_sizes): return input_sizes[0] - @config_class class ConvOperator(Operator): type = 'conv' - def __init__( - self, - input_layer_names, - num_filters=None, - conv_conf=None, - **xargs): - super(ConvOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, + input_layer_names, + num_filters=None, + conv_conf=None, + **xargs): + super(ConvOperator, self).__init__(input_layer_names, **xargs) if num_filters is not None: self.operator_conf.num_filters = num_filters parse_conv(conv_conf, MakeLayerNameInSubmodel(input_layer_names[0]), - self.operator_conf.conv_conf) - self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x ** 2) * num_filters + self.operator_conf.conv_conf, num_filters) + self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x + **2) * num_filters config_assert(len(input_layer_names) == 2, "Conv is binary operator") @@ -699,88 +774,106 @@ def calc_output_size(self, input_sizes): # please refer to the comments in proto/ModelConfig.proto @config_class class Conv(Cfg): - def __init__( - self, - filter_size, - channels, - padding = None, - stride = None, - groups = None, - filter_channels = None, - output_x = None, - img_size = None, - caffe_mode = True, - filter_size_y = None, - padding_y = None, - stride_y = None): + def __init__(self, + filter_size, + channels, + padding=None, + stride=None, + groups=None, + filter_channels=None, + output_x=None, + img_size=None, + caffe_mode=True, + filter_size_y=None, + padding_y=None, + stride_y=None): self.add_keys(locals()) if filter_size_y is None: - self.filter_size_y = filter_size + self.filter_size_y = filter_size if padding_y is None: - self.padding_y = padding + self.padding_y = padding if stride_y is None: - self.stride_y = stride + self.stride_y = stride if output_x is not None: - config_assert(output_x <= 0) + config_assert(output_x <= 0) + + +# please refer to the comments in proto/ModelConfig.proto +@config_class +class BilinearInterp(Cfg): + def __init__(self, out_size_x=None, out_size_y=None, num_channels=None): + self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Pool(Cfg): - def __init__( - self, - pool_type, - channels, - size_x, - size_y = None, - img_width = None, - start = None, - stride = None, - stride_y = None, - padding = None, - padding_y = None): + def __init__(self, + pool_type, + channels, + size_x, + size_y=None, + img_width=None, + start=None, + stride=None, + stride_y=None, + padding=None, + padding_y=None): self.add_keys(locals()) + +# please refer to the comments in proto/ModelConfig.proto +@config_class +class SpatialPyramidPool(Cfg): + def __init__(self, pool_type, pyramid_height, channels, img_width=None): + self.add_keys(locals()) + + # please refer to the comments in proto/ModelConfig.proto @config_class class Norm(Cfg): - def __init__( - self, - norm_type, - channels, - size, - scale, - pow, - output_x = None, - img_size = None, - blocked = None): + def __init__(self, + norm_type, + channels, + size, + scale, + pow, + output_x=None, + img_size=None, + blocked=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Image(Cfg): - def __init__( - self, - channels, - img_size = None): + def __init__(self, channels, img_size=None): self.add_keys(locals()) + @config_class class BlockExpand(Cfg): - def __init__( - self, - channels, - padding_x = 0, - padding_y = 0, - stride_x = 0, - stride_y = 0, - block_x = 0, - block_y = 0, - img_size_x = 0, - img_size_y = 0, - output_x = 0, - output_y = 0): + def __init__(self, + channels, + padding_x=0, + padding_y=0, + stride_x=0, + stride_y=0, + block_x=0, + block_y=0, + img_size_x=0, + img_size_y=0, + output_x=0, + output_y=0): + self.add_keys(locals()) + + +@config_class +class MaxOut(Cfg): + def __init__(self, channels, groups, img_size_x=0, img_size_y=0): self.add_keys(locals()) + def DataBase(async_load_data=False, constant_slots=None, data_ratio=1, @@ -794,23 +887,23 @@ def DataBase(async_load_data=False, if constant_slots: data_config.constant_slots.extend(constant_slots) - data_config.data_ratio=data_ratio - data_config.is_main_data=is_main_data + data_config.data_ratio = data_ratio + data_config.is_main_data = is_main_data - usage_ratio=default(usage_ratio, settings_deprecated["usage_ratio"]) + usage_ratio = default(usage_ratio, settings_deprecated["usage_ratio"]) config_assert(usage_ratio >= 0 and usage_ratio <= 1, "The range of usage_ratio is [0, 1]") data_config.usage_ratio = usage_ratio return data_config + @config_func -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None, - **xargs): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'simple' data_config.files = files @@ -821,31 +914,36 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + @config_func -def PyData( - files=None, - type=None, - file_group_queue_capacity=None, - load_data_module=None, - load_data_object=None, - load_data_args="", - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def PyData(files=None, + type=None, + file_group_queue_capacity=None, + load_data_module=None, + load_data_object=None, + load_data_args="", + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'py' if load_data_module in g_py_module_name_list: + def get_path(module): m = __import__(load_data_module) return os.path.split(os.path.realpath(m.__file__))[0] + # python C-api is not thread safe, one module can only be import once, # so here we nedd to copy the module with different names if it has to be # imported several times. - module_new_name = "%s_copy_%d" % (load_data_module, len(g_py_module_name_list)) + module_new_name = "%s_copy_%d" % (load_data_module, + len(g_py_module_name_list)) g_py_module_name_list.append(module_new_name) - module_path = "%s/%s.py" % (get_path(load_data_module), load_data_module) - new_module_path = "%s/%s.py" % (get_path(load_data_module), module_new_name) + module_path = "%s/%s.py" % (get_path(load_data_module), + load_data_module) + new_module_path = "%s/%s.py" % (get_path(load_data_module), + module_new_name) if os.path.isfile(module_path) == False: raise Exception("File %s is not exist." % module_path) shutil.copy2(module_path, new_module_path) @@ -870,15 +968,15 @@ def get_path(module): data_config.constant_slots.extend(constant_slots) return data_config + @config_func -def ProtoData( - files=None, - type=None, - file_group_queue_capacity=None, - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def ProtoData(files=None, + type=None, + file_group_queue_capacity=None, + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) if type is None: data_config.type = 'proto' @@ -899,25 +997,24 @@ def ProtoData( data_config.constant_slots.extend(constant_slots) return data_config + #real data for training is actually provided by "sub_data" data providers. @config_func -def MultiData( - sub_data=[] - ): +def MultiData(sub_data=[]): data_config = DataConfig() data_config.type = 'multi' data_config.sub_data_configs.extend(sub_data) return data_config + @config_func -def Data( - type, - files=None, - feat_dim=None, - slot_dims=None, - context_len=None, - buffer_capacity=None, - **xargs): +def Data(type, + files=None, + feat_dim=None, + slot_dims=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = type @@ -953,71 +1050,109 @@ def TestData(data_config, async_load_data=None): " Data definition") g_config.test_data_config.async_load_data = async_load_data + +def parse_bilinear(bilinear, input_layer_name, bilinear_conf): + bilinear_conf.out_size_x = bilinear.out_size_x + bilinear_conf.out_size_y = bilinear.out_size_y + bilinear_conf.num_channels = bilinear.num_channels + + +''' +caffe_mode: compute the output size using floor instead of ceil, + which is consistent of caffe and CuDNN's convention. +''' + + +def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): + output = (2 * padding + img_size - filter_size) / float(stride) + if caffe_mode: + return 1 + int(math.floor(output)) + else: + return 1 + int(math.ceil(output)) + + +''' +calcualte image_size based on output_size for convolution. +It is the reverse function of cnn_output_size +''' + + +def cnn_image_size(output_size, filter_size, padding, stride, caffe_mode): + if caffe_mode: + img_size = (output_size - 1) * stride + filter_size - 2 * padding + else: + img_size = (output_size - 2) * stride + filter_size - 2 * padding + 1 + return img_size + + def parse_pool(pool, input_layer_name, pool_conf): pool_conf.pool_type = pool.pool_type - config_assert(pool.pool_type in ['max-projection', 'avg-projection', - 'cudnn-max-pool', 'cudnn-avg-pool'], - "pool-type %s is not in " + config_assert(pool.pool_type in [ + 'max-projection', 'avg-projection', 'cudnn-max-pool', 'cudnn-avg-pool' + ], "pool-type %s is not in " "['max-projection', 'avg-projection', " - "'cudnn-max-pool', 'cudnn-avg-pool']" - % pool.pool_type) - if pool.size_y or pool.stride_y or pool.img_width or pool.padding_y: - config_assert(pool.pool_type.startswith('cudnn'), - "'size_y', 'stride_y' and 'img_width' and 'padding_y'" - "can only be used for cudnn") + "'cudnn-max-pool', 'cudnn-avg-pool']" % pool.pool_type) pool_conf.channels = pool.channels pool_conf.size_x = pool.size_x pool_conf.stride = pool.stride pool_conf.size_y = default(pool.size_y, pool_conf.size_x) - pool_conf.stride_y = default(pool.stride_y, pool_conf.stride); + pool_conf.stride_y = default(pool.stride_y, pool_conf.stride) img_pixels = g_layer_map[input_layer_name].size / pool.channels - pool_conf.img_size = default(pool.img_width, int(img_pixels ** 0.5)) + # the img_width may be removed, + # and it can be calculated automatically later. + pool_conf.img_size = default(pool.img_width, int(img_pixels**0.5)) pool_conf.img_size_y = img_pixels / pool_conf.img_size config_assert(pool_conf.img_size * pool_conf.img_size_y == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (pool_conf.img_size, img_pixels)) - - if pool.start is not None: - config_assert(pool.padding is None, - 'At most one of start and padding can be set.') - pool_conf.start = pool.start - pool_conf.padding = 0 - pool_conf.output_x = int(math.ceil((pool_conf.img_size - \ - pool_conf.start - pool_conf.size_x) / \ - float(pool_conf.stride))) + 1 - - pool_conf.output_y = int(math.ceil((pool_conf.img_size_y - \ - pool_conf.start - pool_conf.size_y) / \ - float(pool_conf.stride_y))) + 1 - elif pool.padding is not None: + "Incorrect input image size %d for input image pixels %d" % + (pool_conf.img_size, img_pixels)) + + config_assert(not pool.start, "start is deprecated in pooling.") + + if pool.padding is not None: pool_conf.padding = pool.padding pool_conf.padding_y = default(pool.padding_y, pool_conf.padding) - pool_conf.start = 0 - pool_conf.output_x = int(math.ceil((pool_conf.img_size + \ - 2*pool_conf.padding - pool_conf.size_x) / \ - float(pool_conf.stride))) + 1 - pool_conf.output_y = int(math.ceil((pool_conf.img_size_y + \ - 2*pool_conf.padding_y - pool_conf.size_y) / \ - float(pool_conf.stride_y))) + 1 - else: - raise ValueError('At least one of start and padding should be set.') + pool_conf.output_x = cnn_output_size( + pool_conf.img_size, pool_conf.size_x, pool_conf.padding, + pool_conf.stride, False) + pool_conf.output_y = cnn_output_size( + pool_conf.img_size_y, pool_conf.size_y, pool_conf.padding_y, + pool_conf.stride_y, False) + + +def parse_spp(spp, input_layer_name, spp_conf): + spp_conf.pool_type = spp.pool_type + config_assert(spp.pool_type in ['max-projection', 'avg-projection'], + "pool-type %s is not in " + "['max-projection', 'avg-projection']" % spp.pool_type) + spp_conf.pyramid_height = spp.pyramid_height + spp_conf.channels = spp.channels + + img_pixels = g_layer_map[input_layer_name].size / spp_conf.channels + + spp_conf.img_size = default(spp.img_width, int(img_pixels**0.5)) + spp_conf.img_size_y = img_pixels / spp_conf.img_size + config_assert(spp_conf.img_size * spp_conf.img_size_y == img_pixels, + "Incorrect input image size %d for input image pixels %d" % + (spp_conf.img_size, img_pixels)) + def parse_image(image, input_layer_name, image_conf): image_conf.channels = image.channels image_pixels = g_layer_map[input_layer_name].size / image_conf.channels - image_conf.img_size = int(image_pixels ** 0.5) - config_assert((image_conf.img_size ** 2) == image_pixels, - "Incorrect input image size %d for input image pixels %d" - % (image_conf.img_size, image_pixels)) + image_conf.img_size = int(image_pixels**0.5) + config_assert((image_conf.img_size**2) == image_pixels, + "Incorrect input image size %d for input image pixels %d" % + (image_conf.img_size, image_pixels)) + def parse_norm(norm, input_layer_name, norm_conf): norm_conf.norm_type = norm.norm_type config_assert(norm.norm_type in ['rnorm', 'cmrnorm-projection'], - "norm-type %s is not in [rnorm, 'cmrnorm-projection']" - % norm.norm_type) + "norm-type %s is not in [rnorm, 'cmrnorm-projection']" % + norm.norm_type) norm_conf.channels = norm.channels norm_conf.size = norm.size norm_conf.scale = norm.scale @@ -1025,20 +1160,24 @@ def parse_norm(norm, input_layer_name, norm_conf): norm_conf.blocked = norm.blocked img_pixels = g_layer_map[input_layer_name].size / norm.channels - norm_conf.img_size = int(img_pixels ** 0.5) - config_assert((norm_conf.img_size ** 2) == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (norm_conf.img_size, img_pixels)) + norm_conf.img_size = int(img_pixels**0.5) + config_assert((norm_conf.img_size**2) == img_pixels, + "Incorrect input image size %d for input image pixels %d" % + (norm_conf.img_size, img_pixels)) norm_conf.output_x = norm_conf.img_size if norm.norm_type in ['cmrnorm-projection']: norm_conf.scale /= norm.size else: - norm_conf.scale /= norm.size ** 2 + norm_conf.scale /= norm.size**2 + + ''' caffe_mode: compute the output size using floor instead of ceil, which is consistent of caffe and CuDNN's convention. ''' -def parse_conv(conv, input_layer_name, conv_conf): + + +def parse_conv(conv, input_layer_name, conv_conf, num_filters, trans=False): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y conv_conf.channels = conv.channels @@ -1047,25 +1186,38 @@ def parse_conv(conv, input_layer_name, conv_conf): conv_conf.stride = conv.stride conv_conf.stride_y = conv.stride_y conv_conf.groups = conv.groups - conv_conf.filter_channels = conv.channels / conv.groups conv_conf.caffe_mode = conv.caffe_mode - img_pixels = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.img_size = int(img_pixels ** 0.5) - config_assert((conv_conf.img_size ** 2) == img_pixels, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.img_size, img_pixels)) - if conv.caffe_mode: - conv_conf.output_x = \ - 1 + int(math.floor((2 * conv.padding + conv_conf.img_size \ - - conv.filter_size) / float(conv.stride))) + if not trans: + conv_conf.filter_channels = conv.channels / conv.groups + + img_pixels = g_layer_map[input_layer_name].size / conv.channels + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.img_size = int(img_pixels**0.5) + config_assert((conv_conf.img_size**2) == img_pixels, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.img_size, img_pixels)) + + conv_conf.output_x = cnn_output_size( + conv_conf.img_size, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) else: - conv_conf.output_x = \ - 1 + int(math.ceil((2 * conv.padding + conv_conf.img_size \ - - conv.filter_size) / float(conv.stride))) + conv_conf.filter_channels = num_filters / conv.groups + + outputSize = g_layer_map[input_layer_name].size / conv.channels + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.output_x = int(outputSize**0.5) + config_assert((conv_conf.output_x**2) == outputSize, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.output_x, outputSize)) + conv_conf.img_size = cnn_image_size( + conv_conf.output_x, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) + def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels @@ -1080,18 +1232,24 @@ def parse_block_expand(block_expand, input_layer_name, block_expand_conf): if block_expand_conf.img_size_x == 0: block_expand_conf.output_x = 0 else: - block_expand_conf.output_x = \ - 1 + \ - int(math.ceil((2 * block_expand.padding_x + block_expand.img_size_x \ - - block_expand.block_x) / float(block_expand.stride_x))) + block_expand_conf.output_x = cnn_output_size( + block_expand.img_size_x, block_expand.block_x, + block_expand.padding_x, block_expand.stride_x, False) if block_expand_conf.img_size_y == 0: - block_expand_conf.output_y = 0 + block_expand_conf.output_y = 0 else: - block_expand_conf.output_y = \ - 1 + \ - int(math.ceil((2 * block_expand.padding_y + block_expand.img_size_y \ - - block_expand.block_y) / float(block_expand.stride_y))) + block_expand_conf.output_y = cnn_output_size( + block_expand.img_size_y, block_expand.block_y, + block_expand.padding_y, block_expand.stride_y, False) + + +def parse_maxout(maxout, input_layer_name, maxout_conf): + maxout_conf.channels = maxout.channels + maxout_conf.groups = maxout.groups + maxout_conf.img_size_x = maxout.img_size_x + maxout_conf.img_size_y = maxout.img_size_y + # Define an evaluator @config_func @@ -1099,15 +1257,14 @@ def Evaluator( name, type, inputs, - chunk_scheme = None, - num_chunk_types = None, - classification_threshold = None, - positive_label = None, - dict_file = None, - result_file = None, - num_results = None, - delimited = None, - ): + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None, ): evaluator = g_config.model_config.evaluators.add() evaluator.type = type evaluator.name = MakeLayerNameInSubmodel(name) @@ -1136,19 +1293,20 @@ def Evaluator( if delimited is not None: evaluator.delimited = delimited + class LayerBase(object): def __init__( self, name, type, - size, # size can be 0. In this case, subclass should set it. + size, # size can be 0. In this case, subclass should set it. inputs, device=None, active_type="", drop_rate=0., coeff=None): config_assert('@' not in name, - "layer name: %s contain special character @" % name) + "layer name: %s contain special character @" % name) global g_current_submodel name = MakeLayerNameInSubmodel(name) @@ -1187,8 +1345,8 @@ def __init__( if type_of(input) == str: input_layer_name = input input_config = Input( - input_layer_name = input, - parameter_name = gen_parameter_name(name, input_index)) + input_layer_name=input, + parameter_name=gen_parameter_name(name, input_index)) input_layer_name = input_config.input_layer_name elif isinstance(input, Input): input_layer_name = input.input_layer_name @@ -1197,16 +1355,15 @@ def __init__( input_config.parameter_name = \ gen_parameter_name(name, input_index) elif isinstance(input, Operator): - self.operators.append(input); + self.operators.append(input) input.operator_conf.input_indices.append(input_index) input_config = Input(input.input_layer_names[0]) input_layer_name = input_config.input_layer_name else: - raise ValueError( - 'Wrong type for inputs: %s' % type_of(input)) + raise ValueError('Wrong type for inputs: %s' % type_of(input)) config_assert(input_layer_name in g_layer_map, - "Unknown input layer '%s' for layer %s" - % (input_layer_name, name)) + "Unknown input layer '%s' for layer %s" % + (input_layer_name, name)) self.inputs[input_index] = input_config layer_input = self.config.inputs.add() layer_input.input_layer_name = input_config.input_layer_name @@ -1218,26 +1375,26 @@ def __init__( g_current_submodel.layer_names.append(self.config.name) - def get_input_layer(self, input_index): return g_layer_map[self.config.inputs[input_index].input_layer_name] # will return the bias created if not *for_self* def create_bias_parameter( self, - bias, # True/False or BiasCfg + bias, # True/False or BiasCfg size, - dims = None, - for_self = True, # whether create bias for layer self - ): + dims=None, + for_self=True, # whether create bias for layer self + ): if size == 0: return if dims is None: dims = [1, size] - config_assert(type_of(bias) == bool or type_of(bias) == Bias, - 'Incorrect type for bias: %s' % type_of(bias)) + config_assert( + type_of(bias) == bool or type_of(bias) == Bias, + 'Incorrect type for bias: %s' % type_of(bias)) if type_of(bias) == bool: if bias: @@ -1252,7 +1409,8 @@ def create_bias_parameter( Parameter( bias.parameter_name, size, - self.config.device if self.config.HasField('device') else None, + self.config.device + if self.config.HasField('device') else None, dims, bias.learning_rate, bias.momentum, @@ -1264,22 +1422,21 @@ def create_bias_parameter( initial_smart=bias.initial_smart, num_batches_regularization=bias.num_batches_regularization, sparse_remote_update=bias.sparse_remote_update, - gradient_clipping_threshold=bias.gradient_clipping_threshold, + gradient_clipping_threshold=bias. + gradient_clipping_threshold, is_static=bias.is_static, - is_shared=bias.is_shared, - ) + is_shared=bias.is_shared, ) if for_self: self.config.bias_parameter_name = bias.parameter_name else: return bias.parameter_name - def create_input_parameter( - self, - input_index, - size, - dims=None, - sparse = None, - format = None): + def create_input_parameter(self, + input_index, + size, + dims=None, + sparse=None, + format=None): if dims is None: # TODO(yuyang18): print warning and callstack here! dims = list() @@ -1294,12 +1451,12 @@ def create_input_parameter( if input_config.parameter_name in g_parameter_map: para = g_parameter_map[input_config.parameter_name] - config_assert(size == para.size, ('Shared parameter "%s" does not ' - + 'have same size: %s vs. %s') + config_assert(size == para.size, ( + 'Shared parameter "%s" does not ' + 'have same size: %s vs. %s') % (input_config.parameter_name, para.size, size)) - config_assert(dims == para.dims, ('Shared parameter "%s" does not ' - + 'have same dims: %s vs. %s') + config_assert(dims == para.dims, ( + 'Shared parameter "%s" does not ' + 'have same dims: %s vs. %s') % (input_config.parameter_name, para.dims, dims)) return @@ -1319,13 +1476,13 @@ def create_input_parameter( num_batches_regularization=input_config.num_batches_regularization, sparse_remote_update=input_config.sparse_remote_update, sparse_update=input_config.sparse_update, - gradient_clipping_threshold=input_config.gradient_clipping_threshold, + gradient_clipping_threshold=input_config. + gradient_clipping_threshold, sparse=sparse, format=format, is_static=input_config.is_static, is_shared=input_config.is_shared, - update_hooks=input_config.update_hooks - ) + update_hooks=input_config.update_hooks) def set_layer_size(self, size): if self.config.size == 0: @@ -1335,27 +1492,18 @@ def set_layer_size(self, size): 'Different inputs result in' + 'different layer size at layer %s' % self.config.name) + @config_layer('multi_class_cross_entropy_with_selfnorm') class MultiClassCrossEntropySelfNormCostLayer(LayerBase): - def __init__( - self, - name, - inputs, - softmax_selfnorm_alpha=0.1, - **xargs): - super(MultiClassCrossEntropySelfNormCostLayer, self).__init__(name, - 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) + def __init__(self, name, inputs, softmax_selfnorm_alpha=0.1, **xargs): + super(MultiClassCrossEntropySelfNormCostLayer, self).__init__( + name, 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) self.config.softmax_selfnorm_alpha = softmax_selfnorm_alpha + @config_layer('fc') class FCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - **xargs): + def __init__(self, name, size, inputs, bias=True, **xargs): super(FCLayer, self).__init__(name, 'fc', size, inputs=inputs, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) @@ -1369,22 +1517,23 @@ def __init__( else: sparse = None - self.create_input_parameter(input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('selective_fc') class SelectiveFCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - selective_fc_pass_generation=False, - has_selected_colums=True, - selective_fc_full_mul_ratio=0.02, - selective_fc_parallel_plain_mul_thread_num=None, - **xargs): + def __init__(self, + name, + size, + inputs, + bias=True, + selective_fc_pass_generation=False, + has_selected_colums=True, + selective_fc_full_mul_ratio=0.02, + selective_fc_parallel_plain_mul_thread_num=None, + **xargs): super(SelectiveFCLayer, self).__init__( name, 'selective_fc', size, inputs=inputs, **xargs) # user MUST know if selctive fc is used in training, @@ -1405,8 +1554,8 @@ def __init__( input_num = len(self.inputs) if has_selected_colums: config_assert(input_num >= 2, - ("if indices of selected columns are not specified, " - "selective_fc Layer has at least two inputs")) + ("if indices of selected columns are not specified, " + "selective_fc Layer has at least two inputs")) input_num -= 1 for input_index in xrange(input_num): @@ -1419,26 +1568,23 @@ def __init__( if sparse: psize = self.inputs[input_index].nnz - self.create_input_parameter( - input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('print') class PrintLayer(LayerBase): - def __init__( - self, - name, - inputs): + def __init__(self, name, inputs): super(PrintLayer, self).__init__(name, 'print', 0, inputs) + @config_layer('data') class DataLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(DataLayer, self).__init__(name, 'data' , size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(DataLayer, self).__init__( + name, 'data', size, inputs=[], device=device) + ''' DataNormLayer: A layer for data normalization @@ -1466,14 +1612,11 @@ def __init__( min-max: y = (x-min)/(max-min) decimal-scaling: y = x/10^j, where j is the smallest integer such that max(|y|)<1 ''' + + @config_layer('data_norm') class DataNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - data_norm_strategy="z-score", - device=None): + def __init__(self, name, inputs, data_norm_strategy="z-score", device=None): super(DataNormLayer, self).__init__( name, 'data_norm', 0, inputs=inputs, device=device) self.config.data_norm_strategy = data_norm_strategy @@ -1485,15 +1628,12 @@ def __init__( self.inputs[0].is_static = True self.create_input_parameter(0, para_size, para_dims) + @config_layer('prelu') class ParameterReluLayer(LayerBase): layer_type = 'prelu' - def __init__( - self, - name, - inputs, - partial_sum = 1, - **args): + + def __init__(self, name, inputs, partial_sum=1, **args): super(ParameterReluLayer, self).__init__( name, self.layer_type, 0, inputs=inputs, **args) config_assert(len(self.inputs) == 1) @@ -1502,17 +1642,18 @@ def __init__( self.set_layer_size(input_layer.size) self.create_input_parameter(0, input_layer.size / partial_sum) + @config_layer('conv') class ConvLayerBase(LayerBase): layer_type = 'conv' - def __init__( - self, - name, - inputs=[], - bias=True, - num_filters=None, - shared_biases=False, - **xargs): + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): super(ConvLayerBase, self).__init__( name, self.layer_type, 0, inputs=inputs, **xargs) @@ -1529,7 +1670,7 @@ def __init__( config_assert(use_gpu, "cudnn_conv only support GPU") if (use_gpu == 1 and self.layer_type != "exconv" and - (parallel_nn == 0 or self.config.device > -1)): + (parallel_nn == 0 or self.config.device > -1)): self.layer_type = "cudnn_conv" else: self.layer_type = "exconv" @@ -1541,16 +1682,14 @@ def __init__( for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_conv( - self.inputs[input_index].conv, - input_layer.name, - self.config.inputs[input_index].conv_conf) + parse_conv(self.inputs[input_index].conv, input_layer.name, + self.config.inputs[input_index].conv_conf, num_filters) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) print("output size for %s is %d " % (name, conv_conf.output_x)) self.create_input_parameter(input_index, psize) self.set_layer_size( - (conv_conf.output_x ** 2) * self.config.num_filters) + (conv_conf.output_x**2) * self.config.num_filters) psize = self.config.size if shared_biases: @@ -1561,70 +1700,139 @@ def calc_parameter_size(self, conv_conf): return self.config.num_filters * conv_conf.filter_channels \ * (conv_conf.filter_size * conv_conf.filter_size_y) + @config_layer('exconv') class ConvLayer(ConvLayerBase): layer_type = 'exconv' + @config_layer('cudnn_conv') class ConvLayer(ConvLayerBase): layer_type = 'cudnn_conv' -@config_layer('norm') -class NormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(NormLayer, self).__init__(name, 'norm', 0, inputs=inputs, device=device) + +@config_layer('convt') +class ConvTransLayerBase(LayerBase): + layer_type = 'convt' + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): + super(ConvTransLayerBase, self).__init__( + name, self.layer_type, 0, inputs=inputs, **xargs) + + if num_filters is not None: + self.config.num_filters = num_filters + + use_gpu = int(g_command_config_args.get("use_gpu", 0)) + parallel_nn = int(g_command_config_args.get("parallel_nn", 0)) + + # cudnn_convt has not been implemented so use exconvt only + self.layer_type = "exconvt" + # need to specify layer in config + self.config.type = self.layer_type + + if shared_biases is not None: + self.config.shared_biases = shared_biases + for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_norm( - self.inputs[input_index].norm, + parse_conv( + self.inputs[input_index].conv, input_layer.name, - self.config.inputs[input_index].norm_conf) - norm_conf = self.config.inputs[input_index].norm_conf - self.set_layer_size((norm_conf.output_x ** 2) * norm_conf.channels) - -@config_layer('pool') + self.config.inputs[input_index].conv_conf, + num_filters, + trans=True) + conv_conf = self.config.inputs[input_index].conv_conf + psize = self.calc_parameter_size(conv_conf) + print("output size for %s is %d " % (name, conv_conf.output_x)) + self.create_input_parameter(input_index, psize) + self.set_layer_size( + (conv_conf.img_size**2) * self.config.num_filters) + + psize = self.config.size + if shared_biases: + psize = self.config.num_filters + self.create_bias_parameter(bias, psize, [psize, 1]) + + def calc_parameter_size(self, conv_conf): + return conv_conf.channels * conv_conf.filter_channels \ + * (conv_conf.filter_size * conv_conf.filter_size_y) + + +@config_layer('exconvt') +class ConvTransLayer(ConvTransLayerBase): + layer_type = 'exconvt' + + +@config_layer('norm') +class NormLayer(LayerBase): + def __init__(self, name, inputs, device=None): + super(NormLayer, self).__init__( + name, 'norm', 0, inputs=inputs, device=device) + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + parse_norm(self.inputs[input_index].norm, input_layer.name, + self.config.inputs[input_index].norm_conf) + norm_conf = self.config.inputs[input_index].norm_conf + self.set_layer_size((norm_conf.output_x**2) * norm_conf.channels) + + +@config_layer('pool') class PoolLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PoolLayer, self).__init__(name, 'pool', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PoolLayer, self).__init__( + name, 'pool', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_pool( - self.inputs[input_index].pool, - input_layer.name, - self.config.inputs[input_index].pool_conf) + parse_pool(self.inputs[input_index].pool, input_layer.name, + self.config.inputs[input_index].pool_conf) pool_conf = self.config.inputs[input_index].pool_conf - print("output size for %s is %d*%d " % ( - name, pool_conf.output_y, pool_conf.output_x)) - self.set_layer_size((pool_conf.output_x ** 2) * pool_conf.channels) + print("output size for %s is %d*%d " % (name, pool_conf.output_y, + pool_conf.output_x)) + self.set_layer_size( + (pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) + + +@config_layer('spp') +class SpatialPyramidPoolLayer(LayerBase): + def __init__(self, name, inputs, device=None): + super(SpatialPyramidPoolLayer, self).__init__( + name, 'spp', 0, inputs=inputs, device=device) + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + parse_spp(self.inputs[input_index].spp, input_layer.name, + self.config.inputs[input_index].spp_conf) + spp_conf = self.config.inputs[input_index].spp_conf + output_size = (pow(4, spp_conf.pyramid_height) - 1) / (4 - 1) + print("output size for %s is %d " % (name, output_size)) + self.set_layer_size(output_size * spp_conf.channels) + @config_layer('batch_norm') class BatchNormLayer(LayerBase): layer_type = 'batch_norm' - def __init__( - self, - name, - inputs, - active_type="linear", - bias=True, - device=None, - use_global_stats=True, - moving_average_fraction=0.9, - batch_norm_type=None, - **xargs): + + def __init__(self, + name, + inputs, + active_type="linear", + bias=True, + device=None, + use_global_stats=True, + moving_average_fraction=0.9, + batch_norm_type=None, + **xargs): if inputs is None: inputs = [] elif not isinstance(inputs, list): inputs = [inputs] - config_assert(len(inputs) == 1, - "BatchNormLayer must have one and only one input") + config_assert( + len(inputs) == 1, "BatchNormLayer must have one and only one input") # Create Input for moving mean and std, # in batch normalization layer. # These paras no need to update, so set is_static is true. @@ -1633,12 +1841,13 @@ def __init__( use_gpu = bool(int(g_command_config_args.get("use_gpu", 0))) is_shared = True if not use_gpu else False for i in xrange(2): - inputs.append(Input(inputs[0].input_layer_name, - initial_std=0.0, - initial_mean=0.0, - is_static=True, - is_shared=is_shared, - )) + inputs.append( + Input( + inputs[0].input_layer_name, + initial_std=0.0, + initial_mean=0.0, + is_static=True, + is_shared=is_shared, )) parallel_nn = bool(int(g_command_config_args.get("parallel_nn", 0))) cudnn_version = int(g_command_config_args.get("cudnn_version", 0)) @@ -1648,21 +1857,25 @@ def __init__( ((not parallel_nn) or self.config.device > -1) and \ cudnn_version >= 4007 self.layer_type = "cudnn_batch_norm" if use_cudnn else "batch_norm" - super(BatchNormLayer, self).__init__(name, self.layer_type, 0, - active_type=active_type, - inputs=inputs, device=device, **xargs) + super(BatchNormLayer, self).__init__( + name, + self.layer_type, + 0, + active_type=active_type, + inputs=inputs, + device=device, + **xargs) if use_global_stats is not None: self.config.use_global_stats = use_global_stats if moving_average_fraction is not None: self.config.moving_average_fraction = moving_average_fraction - input_layer= self.get_input_layer(0) - parse_image(self.inputs[0].image, - input_layer.name, + input_layer = self.get_input_layer(0) + parse_image(self.inputs[0].image, input_layer.name, self.config.inputs[0].image_conf) image_conf = self.config.inputs[0].image_conf - self.set_layer_size((image_conf.img_size ** 2) * image_conf.channels) + self.set_layer_size((image_conf.img_size**2) * image_conf.channels) psize = self.calc_parameter_size(image_conf) dims = [1, psize] @@ -1675,62 +1888,75 @@ def __init__( def calc_parameter_size(self, image_conf): return image_conf.channels + @config_layer('trans') class TransLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(TransLayer, self).__init__(name, 'trans', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'TransLayer must have one and only one input') + def __init__(self, name, inputs, device=None): + super(TransLayer, self).__init__( + name, 'trans', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'TransLayer must have one and only one input') self.set_layer_size(self.get_input_layer(0).size) + @config_layer('resize') class ResizeLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): - super(ResizeLayer, self).__init__(name, 'resize', size=size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ResizeLayer must have one and only one input') + def __init__(self, name, size, inputs, device=None): + super(ResizeLayer, self).__init__( + name, 'resize', size=size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'ResizeLayer must have one and only one input') + @config_layer('blockexpand') class BlockExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(BlockExpandLayer, self).__init__(name, 'blockexpand', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(BlockExpandLayer, self).__init__( + name, 'blockexpand', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_block_expand(self.inputs[input_index].block_expand, - input_layer.name, + parse_block_expand( + self.inputs[input_index].block_expand, input_layer.name, self.config.inputs[input_index].block_expand_conf) - block_expand_conf = self.config.inputs[input_index].block_expand_conf - self.set_layer_size(block_expand_conf.block_x * block_expand_conf.block_y - * block_expand_conf.channels) + block_expand_conf = self.config.inputs[ + input_index].block_expand_conf + self.set_layer_size(block_expand_conf.block_x * + block_expand_conf.block_y * + block_expand_conf.channels) + + +@config_layer('maxout') +class MaxOutLayer(LayerBase): + def __init__(self, name, inputs, **xargs): + super(MaxOutLayer, self).__init__( + name, 'maxout', 0, inputs=inputs, **xargs) + input_layer = self.get_input_layer(0) + parse_maxout(self.inputs[0].maxout, input_layer.name, + self.config.inputs[0].maxout_conf) + maxout_conf = self.config.inputs[0].maxout_conf + self.set_layer_size(g_layer_map[input_layer.name].size / + maxout_conf.groups) + # key: cost type # value: cost class g_cost_map = {} + # define a cost layer without any parameters def define_cost(class_name, cost_type): def init(cls, name, inputs, device=None, coeff=1.): - super(type(cls), cls).__init__(name, cost_type, 1, inputs, device=device, coeff=coeff) + super(type(cls), cls).__init__( + name, cost_type, 1, inputs, device=device, coeff=coeff) - cls = type(class_name, (LayerBase,), dict(__init__=init)) + cls = type(class_name, (LayerBase, ), dict(__init__=init)) global g_cost_map g_cost_map[cost_type] = cls + define_cost('MultiClassCrossEntropy', 'multi-class-cross-entropy') -define_cost('ClassificationErrorLayer', 'classification_error') define_cost('RankingCost', 'rank-cost') define_cost('AucValidation', 'auc-validation') define_cost('PnpairValidation', 'pnpair-validation') @@ -1738,20 +1964,17 @@ def init(cls, name, inputs, device=None, coeff=1.): define_cost('MultiBinaryLabelCrossEntropy', 'multi_binary_label_cross_entropy') define_cost('SoftBinaryClassCrossEntropy', 'soft_binary_class_cross_entropy') define_cost('HuberTwoClass', 'huber') +define_cost('SumCost', 'sum_cost') + @config_layer('hsigmoid') class HierarchicalSigmoidLayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - device=None, - bias=True): + def __init__(self, name, num_classes, inputs, device=None, bias=True): super(HierarchicalSigmoidLayer, self).__init__( name, 'hsigmoid', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) >= 2, - 'HierarchicalSigmoidLayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, + 'HierarchicalSigmoidLayer must have at least 2 inputs') self.config.num_classes = num_classes for input_index in xrange(len(self.inputs) - 1): input_layer = self.get_input_layer(input_index) @@ -1760,6 +1983,7 @@ def __init__( self.create_input_parameter(input_index, psize, dims) self.create_bias_parameter(bias, num_classes - 1) + ''' lambdaCost for lambdaRank LTR approach @@ -1784,59 +2008,57 @@ def __init__( max_sort_size can be greater than the size of a list, in which case the algorithm will sort the entire list to get gradient. ''' + + @config_layer('lambda_cost') class LambdaCost(LayerBase): - def __init__( - self, - name, - inputs, - NDCG_num = 5, - max_sort_size = -1, - device=None): + def __init__(self, name, inputs, NDCG_num=5, max_sort_size=-1, device=None): super(LambdaCost, self).__init__( name, 'lambda_cost', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'lambdaCost must have 2 inputs') + config_assert(len(self.inputs) == 2, 'lambdaCost must have 2 inputs') self.config.NDCG_num = NDCG_num if max_sort_size != -1: - config_assert(NDCG_num <= max_sort_size, - 'NDCG_num must be less than or equal to max_sort_size') + config_assert( + NDCG_num <= max_sort_size, + 'NDCG_num must be less than or equal to max_sort_size') self.config.max_sort_size = max_sort_size + @config_layer('nce') class NCELayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - num_neg_samples=10, - neg_sampling_dist=None, - bias=True, - **xargs): + def __init__(self, + name, + num_classes, + inputs, + num_neg_samples=10, + neg_sampling_dist=None, + bias=True, + **xargs): super(NCELayer, self).__init__(name, 'nce', 1, inputs=inputs, **xargs) - config_assert(len(self.inputs) >= 2, - 'NCELayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, 'NCELayer must have at least 2 inputs') self.config.num_classes = num_classes if neg_sampling_dist is not None: - config_assert(len(neg_sampling_dist) == num_classes, - 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' - % (len(neg_sampling_dist), num_classes)) + config_assert( + len(neg_sampling_dist) == num_classes, + 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' % + (len(neg_sampling_dist), num_classes)) s = sum(neg_sampling_dist) - config_assert(abs(s - 1) < 1e-5, - 'The sum of neg_sampling_dist (%s) is not 1' % s) + config_assert( + abs(s - 1) < 1e-5, + 'The sum of neg_sampling_dist (%s) is not 1' % s) self.config.neg_sampling_dist.extend(neg_sampling_dist) self.config.num_neg_samples = num_neg_samples num_real_inputs = len(self.inputs) - 1 - input_layer = self.get_input_layer(num_real_inputs) + input_layer = self.get_input_layer(num_real_inputs) config_assert(input_layer.type == 'data', 'Expecting the last input layer of an nce layer to be ' 'a data layer') - if (num_real_inputs > 1 and input_layer.size == 1 - and self.get_input_layer(num_real_inputs - 1).type == 'data'): + if (num_real_inputs > 1 and input_layer.size == 1 and + self.get_input_layer(num_real_inputs - 1).type == 'data'): # This input layer is assumed to be a sample weight layer num_real_inputs -= 1 @@ -1850,105 +2072,82 @@ def __init__( @config_layer('addto') class AddToLayer(LayerBase): - def __init__( - self, - name, - inputs, - bias=True, - **xargs): + def __init__(self, name, inputs, bias=True, **xargs): super(AddToLayer, self).__init__( name, 'addto', 0, inputs=inputs, **xargs) - config_assert(len(inputs) > 0, - 'inputs cannot be empty for AddToLayer') + config_assert(len(inputs) > 0, 'inputs cannot be empty for AddToLayer') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('agent') class AgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(AgentLayer, self).__init__(name, 'agent', size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(AgentLayer, self).__init__( + name, 'agent', size, inputs=[], device=device) + @config_layer('sequence_agent') class SequenceAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceAgentLayer, self).__init__( name, 'sequence_agent', size, inputs=[], device=device) + @config_layer('gather_agent') class GatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(GatherAgentLayer, self).__init__( name, 'gather_agent', size, inputs=[], device=device) + @config_layer('scatter_agent') class ScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(ScatterAgentLayer, self).__init__( name, 'scatter_agent', size, inputs=[], device=device) + @config_layer('sequence_gather_agent') class SequenceGatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceGatherAgentLayer, self).__init__( - name, 'sequence_gather_agent', size, inputs=[], device=device) + name, 'sequence_gather_agent', size, inputs=[], device=device) + @config_layer('sequence_scatter_agent') class SequenceScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceScatterAgentLayer, self).__init__( - name, 'sequence_scatter_agent', size, inputs=[], device=device) + name, 'sequence_scatter_agent', size, inputs=[], device=device) + @config_layer('multiplex') class MultiplexLayer(LayerBase): - def __init__( - self, - name, - inputs, - size, - device=None): - super(MultiplexLayer, self).__init__(name, 'multiplex', size, inputs=inputs, device=device) - config_assert(len(inputs) > 2, - 'MultiplexLayer should have more than 2 inputs.') + def __init__(self, name, inputs, size, device=None): + super(MultiplexLayer, self).__init__( + name, 'multiplex', size, inputs=inputs, device=device) + config_assert( + len(inputs) > 2, 'MultiplexLayer should have more than 2 inputs.') for i in range(1, len(inputs)): - config_assert(self.get_input_layer(i).size == size, - "All the input layers except the first one should" - "have the same size as the MultiplexLayer.") + config_assert( + self.get_input_layer(i).size == size, + "All the input layers except the first one should" + "have the same size as the MultiplexLayer.") + @config_func -def Link(name, - has_subseq=False, - ): +def Link( + name, + has_subseq=False, ): link_config = LinkConfig() link_config.link_name = name link_config.has_subseq = has_subseq return link_config + # memory for recurrent layer group. # *name* and *size* are actual layer's name and size. # will return name of the memory, @@ -1963,43 +2162,46 @@ def Link(name, # can only be initailized by a *boot_layer* which is a sequence. # @config_func -def Memory(name, - size, - is_sequence=False, - boot_layer=None, - boot_bias=False, - boot_bias_active_type="", - boot_with_const_id=None, - ): +def Memory( + name, + size, + is_sequence=False, + boot_layer=None, + boot_bias=False, + boot_bias_active_type="", + boot_with_const_id=None, ): agent_name = name + "+delay1" if is_sequence: agent_layer = SequenceAgentLayer(agent_name, size) else: agent_layer = AgentLayer(agent_name, size) config_assert(g_current_submodel.is_recurrent_layer_group, - 'Memory should be used in recurrent layer group only') + 'Memory should be used in recurrent layer group only') memory = g_current_submodel.memories.add() memory.layer_name = MakeLayerNameInSubmodel(name) memory.link_name = MakeLayerNameInSubmodel(agent_name) memory.is_sequence = is_sequence - options = sum((boot_layer is not None, - bool(boot_bias), + options = sum((boot_layer is not None, bool(boot_bias), boot_with_const_id is not None)) - config_assert(options <= 1, - 'take one option at most from boot_layer, boot_bias, or boot_with_const_id') + config_assert( + options <= 1, + 'take one option at most from boot_layer, boot_bias, or boot_with_const_id' + ) if boot_layer is not None: boot_layer = MakeLayerNameInParentSubmodel(boot_layer) config_assert(boot_layer in g_layer_map, - 'boot_layer "%s" does not correspond to a layer name' % boot_layer) + 'boot_layer "%s" does not correspond to a layer name' % + boot_layer) memory.boot_layer_name = boot_layer elif boot_bias: memory.boot_bias_parameter_name = agent_layer.create_bias_parameter( - boot_bias, size, for_self = False) + boot_bias, size, for_self=False) memory.boot_bias_active_type = boot_bias_active_type elif boot_with_const_id is not None: memory.boot_with_const_id = boot_with_const_id return agent_name + # Generator for recurrent layer group, to use it: # 1. define a id layer as output of layer group # 2. define a memory of this id layer, and assign a boot id(begin of sequence) @@ -2011,11 +2213,10 @@ def Memory(name, @config_func def Generator( max_num_frames, - eos_layer_name = "eos_check", - num_results_per_sample = 1, - beam_size = 1, - log_prob = None, - ): + eos_layer_name="eos_check", + num_results_per_sample=1, + beam_size=1, + log_prob=None, ): generator_config = GeneratorConfig() generator_config.max_num_frames = max_num_frames generator_config.eos_layer_name = eos_layer_name @@ -2025,60 +2226,55 @@ def Generator( generator_config.log_prob = log_prob return generator_config + @config_layer('expand') class ExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - device=None, - bias=False): - super(ExpandLayer, self).__init__( - name, 'expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ExpandLayer takes 2 and only 2 inputs') - self.config.trans_type = trans_type - for input_index in xrange(len(self.inputs)): - input_layer = self.get_input_layer(input_index) - self.set_layer_size(self.get_input_layer(0).size) - self.create_bias_parameter(bias, self.config.size) + def __init__(self, + name, + inputs, + trans_type='non-seq', + device=None, + bias=False): + super(ExpandLayer, self).__init__( + name, 'expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ExpandLayer takes 2 and only 2 inputs') + self.config.trans_type = trans_type + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + self.set_layer_size(self.get_input_layer(0).size) + self.create_bias_parameter(bias, self.config.size) + @config_layer('featmap_expand') class FeatMapExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None, - num_filters=None, - bias=False): - super(FeatMapExpandLayer, self).__init__( - name, 'featmap_expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ExpandLayer takes 1 and only 1 inputs') - if num_filters is not None: + def __init__(self, name, inputs, device=None, num_filters=None, bias=False): + super(FeatMapExpandLayer, self).__init__( + name, 'featmap_expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'ExpandLayer takes 1 and only 1 inputs') + if num_filters is not None: self.config.num_filters = num_filters - else: + else: logger.fatal("FeatMapExpandLayer must specify num_filters.") - self.set_layer_size(self.get_input_layer(0).size * num_filters) + self.set_layer_size(self.get_input_layer(0).size * num_filters) @config_layer('max') class MaxLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - active_type='linear', - device=None, - bias=False, - output_max_index=None): - super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, device=device) + def __init__(self, + name, + inputs, + trans_type='non-seq', + active_type='linear', + device=None, + bias=False, + output_max_index=None): + super(MaxLayer, self).__init__( + name, 'max', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') - self.config.trans_type = trans_type - self.config.active_type = active_type + self.config.trans_type = trans_type + self.config.active_type = active_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2089,12 +2285,7 @@ def __init__( @config_layer('maxid') class MaxIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - beam_size=None, - device=None): + def __init__(self, name, inputs, beam_size=None, device=None): super(MaxIdLayer, self).__init__( name, 'maxid', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxIdLayer must have 1 input') @@ -2112,37 +2303,39 @@ def __init__( @config_layer('eos_id') class EosIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - eos_id, - device=None): + def __init__(self, name, inputs, eos_id, device=None): super(EosIdLayer, self).__init__( name, 'eos_id', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'EosIdLayer must have 1 input') - self.set_layer_size(2) # boolean output + self.set_layer_size(2) # boolean output self.config.eos_id = eos_id + @config_layer('seqlastins') class SequenceLastInstanceLayer(LayerBase): - def __init__( - self, + def __init__(self, + name, + inputs, + active_type='linear', + trans_type='non-seq', + device=None, + bias=False): + super(SequenceLastInstanceLayer, self).__init__( name, - inputs, - active_type='linear', - trans_type='non-seq', - device=None, - bias=False): - super(SequenceLastInstanceLayer, self).__init__(name, 'seqlastins', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') - self.config.trans_type = trans_type + 'seqlastins', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') + self.config.trans_type = trans_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqfirstins') class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): def __init__( @@ -2152,167 +2345,163 @@ def __init__( active_type='linear', trans_type='non-seq', device=None, - bias=False, - ): - super(SequenceFirstInstanceLayer, self).__init__(name, - inputs=inputs, active_type=active_type, device=device, bias=bias) - self.config.trans_type = trans_type + bias=False, ): + super(SequenceFirstInstanceLayer, self).__init__( + name, + inputs=inputs, + active_type=active_type, + device=device, + bias=bias) + self.config.trans_type = trans_type self.config.select_first = True + @config_layer('seqconcat') class SequenceConcatLayer(LayerBase): - def __init__( - self, + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceConcatLayer, self).__init__( name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceConcatLayer, self).__init__(name, 'seqconcat', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') + 'seqconcat', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqreshape') class SequenceReshapeLayer(LayerBase): - def __init__( - self, + def __init__(self, + name, + size, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceReshapeLayer, self).__init__( name, + 'seqreshape', size, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceReshapeLayer, self).__init__(name, 'seqreshape', - size, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('subseq') class SubSequenceLayer(LayerBase): - def __init__( - self, + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SubSequenceLayer, self).__init__( name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SubSequenceLayer, self).__init__(name, 'subseq', - 0, inputs=inputs, device=device, active_type=active_type) + 'subseq', + 0, + inputs=inputs, + device=device, + active_type=active_type) config_assert(len(inputs) == 3, 'SubSequenceLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) size = input_layer0.size self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('out_prod') class OuterProdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(OuterProdLayer, self).__init__(name, 'out_prod', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(OuterProdLayer, self).__init__( + name, 'out_prod', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'OuterProdLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer0.size * input_layer1.size) + @config_layer('power') class PowerLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PowerLayer, self).__init__(name, 'power', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PowerLayer, self).__init__( + name, 'power', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'PowerLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input is the exponent and should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input is the exponent and should be of size 1') + @config_layer('slope_intercept') class SlopeInterceptLayer(LayerBase): - def __init__( - self, - name, - inputs, - slope=1.0, - intercept=0.0, - device=None): - super(SlopeInterceptLayer, self).__init__(name, 'slope_intercept', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, slope=1.0, intercept=0.0, device=None): + super(SlopeInterceptLayer, self).__init__( + name, 'slope_intercept', 0, inputs=inputs, device=device) self.config.slope = slope self.config.intercept = intercept config_assert(len(inputs) == 1, 'SlopeInterceptLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('scaling') class ScalingLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ScalingLayer, self).__init__(name, 'scaling', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ScalingLayer, self).__init__( + name, 'scaling', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ScalingLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input should be of size 1') + @config_layer('conv_shift') class ConvShiftLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ConvShiftLayer, self).__init__(name, 'conv_shift', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ConvShiftLayer, self).__init__( + name, 'conv_shift', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ConvShiftLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('convex_comb') class ConvexCombinationLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(ConvexCombinationLayer, self).__init__( - name, 'convex_comb', size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ConvexCombinationLayer must have 2 inputs') + name, 'convex_comb', size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ConvexCombinationLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for ConvexCombinationLayer') self.set_layer_size(size) + @config_layer('interpolation') class InterpolationLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(InterpolationLayer, self).__init__( name, 'interpolation', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 3, - 'InterpolationLayer must have 3 inputs') + config_assert( + len(self.inputs) == 3, 'InterpolationLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) input_layer2 = self.get_input_layer(2) @@ -2321,48 +2510,51 @@ def __init__( config_assert(input_layer1.size == input_layer2.size, 'the two vector inputs should be of the same size') + +@config_layer('bilinear_interp') +class BilinearInterpLayer(LayerBase): + def __init__(self, name, inputs, **xargs): + super(BilinearInterpLayer, self).__init__( + name, 'bilinear_interp', 0, inputs=inputs, **xargs) + input_layer = self.get_input_layer(0) + parse_bilinear(self.inputs[0].bilinear_interp, input_layer.name, + self.config.inputs[0].bilinear_interp_conf) + conf = self.inputs[0].bilinear_interp + self.set_layer_size(conf.out_size_x * conf.out_size_y * + conf.num_channels) + + @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SumToOneNormLayer, self).__init__( - name, 'sum_to_one_norm', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'SumToOneNormLayer must have 1 input') + name, 'sum_to_one_norm', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'SumToOneNormLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('cos_vm') class CosSimVecMatLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - cos_scale=1.0, - device=None): + def __init__(self, name, size, inputs, cos_scale=1.0, device=None): super(CosSimVecMatLayer, self).__init__( - name, 'cos_vm', size, inputs=inputs, device=device) + name, 'cos_vm', size, inputs=inputs, device=device) self.config.cos_scale = cos_scale - config_assert(len(self.inputs) == 2, - 'CosSimVecMatLayer must have 2 inputs') + config_assert( + len(self.inputs) == 2, 'CosSimVecMatLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for CosSimVecMatLayer') + @config_layer('sampling_id') class SamplingIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SamplingIdLayer, self).__init__( name, 'sampling_id', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') + config_assert( + len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2375,33 +2567,33 @@ def __init__( # 'squarerootn': sum each sample, but divide by sqrt(sample_num). @config_layer('average') class AverageLayer(LayerBase): - def __init__( - self, + def __init__(self, + name, + inputs, + average_strategy='average', + trans_type='non-seq', + active_type='linear', + device=None, + bias=False): + super(AverageLayer, self).__init__( name, - inputs, - average_strategy='average', - trans_type='non-seq', - active_type='linear', - device=None, - bias=False): - super(AverageLayer, self).__init__(name, 'average', 0, inputs=inputs, - device=device, active_type=active_type) + 'average', + 0, + inputs=inputs, + device=device, + active_type=active_type) self.config.average_strategy = average_strategy - self.config.trans_type = trans_type + self.config.trans_type = trans_type config_assert(len(inputs) == 1, 'AverageLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('cos') class CosSimLayer(LayerBase): - def __init__( - self, - name, - inputs, - cos_scale=5, - device=None): + def __init__(self, name, inputs, cos_scale=5, device=None): super(CosSimLayer, self).__init__( name, 'cos', 1, inputs=inputs, device=device) config_assert(len(self.inputs) == 2, 'CosSimLayer must have 2 inputs') @@ -2413,18 +2605,13 @@ def __init__( @config_layer('tensor') class TensorLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None, - bias=True, - **xargs): - super(TensorLayer, self).__init__(name, 'tensor', size, inputs=inputs, device=device, **xargs) + def __init__(self, name, size, inputs, device=None, bias=True, **xargs): + super(TensorLayer, self).__init__( + name, 'tensor', size, inputs=inputs, device=device, **xargs) config_assert(len(self.inputs) == 2, 'TensorLayer must have 2 inputs') config_assert(size > 0, 'size must be positive') - config_assert(inputs[1].parameter_name == None, 'second parameter should be None.') + config_assert(inputs[1].parameter_name == None, + 'second parameter should be None.') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) psize = size * input_layer0.size * input_layer1.size @@ -2435,14 +2622,13 @@ def __init__( @config_layer('mixed') class MixedLayer(LayerBase): - def __init__( - self, - name, - inputs, - size=0, - bias=True, - error_clipping_threshold=None, - **xargs): + def __init__(self, + name, + inputs, + size=0, + bias=True, + error_clipping_threshold=None, + **xargs): config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( name, 'mixed', size, inputs=inputs, **xargs) @@ -2467,24 +2653,28 @@ def __init__( else: sz = operator.calc_output_size(operator_conf.input_sizes) if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] if input_index not in operator_input_index: - config_assert(isinstance(input, Projection), "input should be projection or operation") + config_assert( + isinstance(input, Projection), + "input should be projection or operation") if self.config.size == 0 and isinstance(input, Projection): size = input.calc_output_size(input_layer) if size != 0: self.set_layer_size(size) elif isinstance(input, Projection): - sz = input.calc_output_size(input_layer) - if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + sz = input.calc_output_size(input_layer) + if sz != 0: + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) config_assert(size != 0, "size is not set") for input_index in xrange(len(self.inputs)): @@ -2496,7 +2686,8 @@ def __init__( input_config = self.config.inputs[input_index] input_config.proj_conf.CopyFrom(input.proj_conf) - input_config.proj_conf.name = gen_parameter_name(name, input_index) + input_config.proj_conf.name = gen_parameter_name(name, + input_index) psize = input.calc_parameter_size(input_layer.size, size) dims = input.calc_parameter_dims(input_layer.size, size) self.create_input_parameter(input_index, psize, dims) @@ -2508,49 +2699,60 @@ def __init__( record_operator_conf = self.config.operator_confs.add() record_operator_conf.CopyFrom(operator_conf) + psize = self.config.size + if isinstance(self.inputs[0], ConvProjection): + self.config.shared_biases = True + psize = 0 + for input in self.inputs: + psize += input.calc_bias_size() - self.create_bias_parameter(bias, self.config.size) + if bias: + self.config.bias_size = psize + self.create_bias_parameter(bias, psize) if error_clipping_threshold is not None: self.config.error_clipping_threshold = error_clipping_threshold + # like MixedLayer, but no bias parameter @config_func -def ExpressionLayer(name, - inputs, - **xargs): +def ExpressionLayer(name, inputs, **xargs): MixedLayer(name, inputs, bias=False, **xargs) + @config_layer('concat') class ConcatenateLayer(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') + config_assert(not bias, 'ConcatenateLayer cannot support bias.') super(ConcatenateLayer, self).__init__( name, 'concat', 0, inputs=inputs, **xargs) size = 0 for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] - if self.config.size == 0: + if self.config.size == 0: size += input_layer.size self.set_layer_size(size) + # like concat layer, but each input layer was processed by a Projection. @config_layer('concat2') class ConcatenateLayer2(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') super(ConcatenateLayer2, self).__init__( name, 'concat2', 0, inputs=inputs, **xargs) + + if isinstance(self.inputs[0], ConvProjection): + for input_index in xrange(len(self.inputs) - 1): + input = self.inputs[input_index + 1] + config_assert( + isinstance(input, ConvProjection), + "The first input of ConcatenateLayer2 is ConvProjection, " + "the other inputs should also be ConvProjection.") + size = 0 for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) @@ -2571,21 +2773,28 @@ def __init__( input_config.proj_conf.CopyFrom(input.proj_conf) input_config.proj_conf.name = gen_parameter_name(name, input_index) psize = input.calc_parameter_size(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) dims = input.calc_parameter_dims(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) self.create_input_parameter(input_index, psize, dims) + psize = self.config.size + if isinstance(self.inputs[0], ConvProjection): + self.config.shared_biases = True + psize = 0 + for input in self.inputs: + psize += input.calc_bias_size() + + if bias: + self.config.bias_size = psize + self.create_bias_parameter(bias, psize) + + @config_layer('recurrent') class RecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - bias=True, - **xargs): - super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, **xargs) + def __init__(self, name, inputs, reversed=False, bias=True, **xargs): + super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, + **xargs) config_assert(len(self.inputs) == 1, 'RecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) size = input_layer.size @@ -2595,17 +2804,17 @@ def __init__( self.create_input_parameter(0, size * size, dims) self.create_bias_parameter(bias, self.config.size) + @config_layer('lstmemory') class LstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): super(LstmLayer, self).__init__(name, 'lstmemory', 0, inputs, **xargs) config_assert(len(self.inputs) == 1, 'LstmLayer must have 1 input') input_layer = self.get_input_layer(0) @@ -2614,117 +2823,126 @@ def __init__( size = input_layer.size / 4 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_input_parameter(0, size * size * 4, [size, size, 4]) #bias includes 3 kinds of peephole, 4 + 3 = 7 self.create_bias_parameter(bias, size * 7) + @config_layer('lstm_step') class LstmStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(LstmStepLayer, self).__init__(name, 'lstm_step', - size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(LstmStepLayer, self).__init__(name, 'lstm_step', size, inputs, + **xargs) config_assert(len(inputs) == 2, 'LstmStepLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 4 * size, 'input_layer0.size != 4 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 4 * size, + 'input_layer0.size != 4 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_bias_parameter(bias, size * 3) + # get the specific output from the input layer. @config_layer('get_output') class GetOutputLayer(LayerBase): - def __init__( - self, - name, - size, - inputs): - super(GetOutputLayer, self).__init__(name, 'get_output' , size, inputs) - config_assert(len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') + def __init__(self, name, size, inputs): + super(GetOutputLayer, self).__init__(name, 'get_output', size, inputs) + config_assert( + len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') inputs = self.inputs[0] config_assert(inputs.input_layer_argument, 'input_layer_argument cannot be empty') + @config_layer('mdlstmemory') class MDLstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - directions=True, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, **xargs) + def __init__(self, + name, + inputs, + directions=True, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, + **xargs) config_assert(len(self.inputs) == 1, 'MDLstmLayer must have 1 input') input_layer = self.get_input_layer(0) dim_num = len(directions) #check input_layer.size is divided by (3+dim_num) - config_assert(input_layer.size % (3+dim_num) == 0, "size % (dim_num) should be 0!") - size = input_layer.size / (3+dim_num) + config_assert(input_layer.size % (3 + dim_num) == 0, + "size % (dim_num) should be 0!") + size = input_layer.size / (3 + dim_num) self.set_layer_size(size) - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type for i in xrange(len(directions)): self.config.directions.append(int(directions[i])) - self.create_input_parameter(0, size * size * (3+dim_num), [size, size, 3+dim_num]) + self.create_input_parameter(0, size * size * (3 + dim_num), + [size, size, 3 + dim_num]) #bias includes 3 kinds of peephole, 3+dim_num+2+dim_num - self.create_bias_parameter(bias, size * (5+2*dim_num)) + self.create_bias_parameter(bias, size * (5 + 2 * dim_num)) + @config_layer('gated_recurrent') class GatedRecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, inputs, **xargs) - config_assert(len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, + inputs, **xargs) + config_assert( + len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) #check input_layer.size is divided by 3 config_assert(input_layer.size % 3 == 0, "size % 3 should be 0!") size = input_layer.size / 3 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + @config_layer('gru_step') class GruStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, + **xargs) config_assert(len(self.inputs) == 2, 'GruStepLayer must have 2 input') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 3 * size, 'input_layer0.size != 3 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 3 * size, + 'input_layer0.size != 3 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + ''' A layer for calculating the cost of sequential conditional random field model. Example: CRFLayer(name="crf_cost", size=label_num, @@ -2732,20 +2950,18 @@ def __init__( where "weight" is optional, one weight for each sequence @param coeff: weight of the layer ''' + + @config_layer('crf') class CRFLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - coeff=1.0, - device=None): + def __init__(self, name, size, inputs, coeff=1.0, device=None): super(CRFLayer, self).__init__(name, 'crf', size, inputs, device=device) - config_assert(2 <= len(self.inputs) <= 3, 'CRFLayer must have 2 or 3 inputs') + config_assert(2 <= len(self.inputs) <= 3, + 'CRFLayer must have 2 or 3 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) self.config.coeff = coeff + ''' A layer for calculating the decoding sequence of sequential conditional random field model. @@ -2754,14 +2970,11 @@ def __init__( this layer will also calculate error, output_.value[i] is 1 for incorrect decoding or 0 for correct decoding ''' + + @config_layer('crf_decoding') class CRFDecodingLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(CRFDecodingLayer, self).__init__( name, 'crf_decoding', size, inputs, device=device) config_assert( @@ -2769,47 +2982,35 @@ def __init__( 'CRFDecodingLayer cannot have more than 2 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) + @config_layer('ctc') class CTCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - norm_by_times = False, - device=None): + def __init__(self, name, size, inputs, norm_by_times=False, device=None): super(CTCLayer, self).__init__(name, 'ctc', size, inputs, device=device) self.config.norm_by_times = norm_by_times config_assert(len(self.inputs) == 2, 'CTCLayer must have 2 inputs') + @config_layer('recurrent_layer_group') class RecurrentLayerGroup(LayerBase): - def __init__( - self, - name, - device=None): + def __init__(self, name, device=None): super(RecurrentLayerGroup, self).__init__( name, 'recurrent_layer_group', 0, inputs=[], device=device) # Deprecated, use a new layer specific class instead @config_func -def Layer( - name, - type, - **xargs): +def Layer(name, type, **xargs): layers = {} layers.update(g_cost_map) layers.update(g_layer_type_map) layer_func = layers.get(type) - config_assert(layer_func, - "layer type '%s' not supported." % type) - layer_func(name, **xargs) + config_assert(layer_func, "layer type '%s' not supported." % type) + return layer_func(name, **xargs) + @config_func -def ParameterHook( - type, - **kwargs): +def ParameterHook(type, **kwargs): if type == 'pruning': mask_filename = kwargs.get('mask_filename', None) assert mask_filename is not None @@ -2822,30 +3023,28 @@ def ParameterHook( @config_func -def Parameter( - name, - size, - device, - dims, - learning_rate=None, - momentum=None, - decay_rate=None, - decay_rate_l1=None, - initial_mean=None, - initial_std=None, - initial_strategy=None, - initial_smart=None, - num_batches_regularization=None, - sparse_remote_update=None, - sparse_update=None, - gradient_clipping_threshold=None, - sparse=None, - format=None, - need_compact=None, - is_static=None, - is_shared=None, - update_hooks=None - ): +def Parameter(name, + size, + device, + dims, + learning_rate=None, + momentum=None, + decay_rate=None, + decay_rate_l1=None, + initial_mean=None, + initial_std=None, + initial_strategy=None, + initial_smart=None, + num_batches_regularization=None, + sparse_remote_update=None, + sparse_update=None, + gradient_clipping_threshold=None, + sparse=None, + format=None, + need_compact=None, + is_static=None, + is_shared=None, + update_hooks=None): config_assert(name not in g_parameter_map, 'Duplicated parameter name: ' + name) @@ -2876,8 +3075,8 @@ def Parameter( para.initial_std = default(initial_std, g_default_initial_std) para.initial_mean = default(initial_mean, g_default_initial_mean) - num_batches_regularization = default( - num_batches_regularization, g_default_num_batches_regularization) + num_batches_regularization = default(num_batches_regularization, + g_default_num_batches_regularization) if num_batches_regularization is not None: para.num_batches_regularization = int(num_batches_regularization) @@ -2887,18 +3086,21 @@ def Parameter( g_config.opt_config.use_sparse_remote_updater = True if sparse_update is not None: para.sparse_update = sparse_update - gradient_clipping_threshold = default( - gradient_clipping_threshold, g_default_gradient_clipping_threshold) + gradient_clipping_threshold = default(gradient_clipping_threshold, + g_default_gradient_clipping_threshold) if gradient_clipping_threshold is not None: para.gradient_clipping_threshold = gradient_clipping_threshold - para.initial_strategy = default(initial_strategy, g_default_initial_strategy) + para.initial_strategy = default(initial_strategy, + g_default_initial_strategy) para.initial_smart = default(initial_smart, g_default_initial_smart) if para.initial_smart: para.initial_mean = 0. if len(para.dims) != 0: para.initial_std = 1. / math.sqrt(para.dims[0]) else: - print("Use initial_smart, but dims not set. Initial_smart may not be used in this layer") + print( + "Use initial_smart, but dims not set. Initial_smart may not be used in this layer" + ) traceback.print_exc() para.initial_std = 1. / math.sqrt(para.size) if g_default_compact_func is not None: @@ -2937,64 +3139,78 @@ def default_initial_std(val): global g_default_initial_std g_default_initial_std = val + @config_func def default_initial_mean(val): global g_default_initial_mean g_default_initial_mean = val + @config_func def default_initial_strategy(val): global g_default_initial_strategy g_default_initial_strategy = val + @config_func def default_initial_smart(val): global g_default_initial_smart g_default_initial_smart = val + @config_func def default_momentum(val): global g_default_momentum g_default_momentum = val + @config_func def default_decay_rate(val): global g_default_decay_rate g_default_decay_rate = val + @config_func def default_num_batches_regularization(val): global g_default_num_batches_regularization g_default_num_batches_regularization = val + @config_func def default_gradient_clipping_threshold(val): global g_default_gradient_clipping_threshold g_default_gradient_clipping_threshold = val + @config_func def default_device(val): global g_default_device g_default_device = val + @config_func def default_update_hooks(val): global g_default_update_hooks g_default_update_hooks = val + @config_func def default_compact_func(val): global g_default_compact_func g_default_compact_func = val + def make_importer(config_dir, config_args): def Import(config_file, local_args={}): if not config_file.startswith('/'): config_file = config_dir + '/' + config_file g_config.config_files.append(config_file) - execfile(config_file, make_config_environment(config_file, config_args), local_args) + execfile(config_file, + make_config_environment(config_file, config_args), local_args) + return Import + settings = dict( batch_size=None, mini_batch_size=None, @@ -3023,26 +3239,24 @@ def Import(config_file, local_args={}): ada_rou=0.95, delta_add_rate=1.0, shrink_parameter_value=0, - adam_beta1 = 0.9, - adam_beta2 = 0.999, - adam_epsilon = 1e-8, -) + adam_beta1=0.9, + adam_beta2=0.999, + adam_epsilon=1e-8, ) -settings_deprecated = dict( - usage_ratio=1., -) +settings_deprecated = dict(usage_ratio=1., ) trainer_settings = dict( save_dir="./output/model", init_model_path=None, - start_pass=0, -) + start_pass=0, ) + @config_func def Settings(**args): for k, v in args.iteritems(): if k == "usage_ratio": - logger.warning("Deprecated: define usage_ratio in DataConfig instead") + logger.warning( + "Deprecated: define usage_ratio in DataConfig instead") if g_config.HasField("data_config"): g_config.data_config.__setattr__(k, v) settings_deprecated[k] = v @@ -3054,10 +3268,12 @@ def Settings(**args): else: logger.fatal('Unkown setting: %s' % k) + @config_func def cluster_config(**args): pass + @config_func def EnableSubmodelSuffix(flag=True): """ @@ -3067,10 +3283,12 @@ def EnableSubmodelSuffix(flag=True): global g_add_submodel_suffix g_add_submodel_suffix = flag + def make_config_environment(config_file, config_args): def make_setter(k): def setter(v): logger.fatal("Obsolete: use Settings(%s=%s, ...) instead" % (k, v)) + return setter funcs = {} @@ -3086,13 +3304,13 @@ def setter(v): funcs.update( Import=make_importer(config_dir, config_args), - get_config_arg=make_get_config_arg(config_args), - ) + get_config_arg=make_get_config_arg(config_args), ) funcs.update(g_extended_config_funcs) return funcs + def make_get_config_arg(config_args): def get_config_arg(name, type, default=None): if type == bool: @@ -3109,6 +3327,7 @@ def get_config_arg(name, type, default=None): return get_config_arg + def importlib(name): __import__(name) return sys.modules[name] @@ -3121,10 +3340,12 @@ def find_caller(): return s[0], s[1], s[2] return "(unknown file)", 0, "(unknown function)" + def my_fatal(s): logger.critical(s) raise Exception() + def parse_config(config_file, config_arg_str): ''' @param config_arg_str: a string of the form var1=val1,var2=val2. It will be @@ -3162,7 +3383,7 @@ def parse_config(config_file, config_arg_str): for k, v in settings.iteritems(): if v is None: continue - g_config.opt_config.__setattr__(k, v); + g_config.opt_config.__setattr__(k, v) for k, v in trainer_settings.iteritems(): if v is None: @@ -3189,6 +3410,7 @@ def parse_config_and_serialize(config_file, config_arg_str): traceback.print_exc() raise + if __name__ == '__main__': try: config = parse_config(sys.argv[1], '') diff --git a/python/paddle/trainer/config_parser_extension.py b/python/paddle/trainer/config_parser_extension.py index 3445076274b0a..ba4c79efdc10e 100644 --- a/python/paddle/trainer/config_parser_extension.py +++ b/python/paddle/trainer/config_parser_extension.py @@ -17,11 +17,10 @@ g_config = None -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None): data_config = DataConfig() data_config.type = 'simple' @@ -33,6 +32,7 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + def get_config_funcs(trainer_config): global g_config g_config = trainer_config diff --git a/python/paddle/trainer/recurrent_units.py b/python/paddle/trainer/recurrent_units.py index 7d51de78b0d79..a80ad13d1ed52 100644 --- a/python/paddle/trainer/recurrent_units.py +++ b/python/paddle/trainer/recurrent_units.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - + # recurrent_units.py # Version 2.0 # @@ -22,161 +22,175 @@ from paddle.trainer.config_parser import * + # long short term memory, can be used in recurrent machine # *inputs* must be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], # *para_prefix* defines parameter names, if the *para_prefix* of # two LstmRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def LstmRecurrentUnit(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): +def LstmRecurrentUnit(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): - if para_prefix is None: + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) LstmStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_check.b"), - inputs = [name + "_" + "input_recurrent", state_memory], - active_type = active_type, - active_gate_type = gate_active_type, - active_state_type = state_active_type, - ) + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_check.b"), + inputs=[name + "_" + "input_recurrent", state_memory], + active_type=active_type, + active_gate_type=gate_active_type, + active_state_type=state_active_type, ) GetOutputLayer( - name = name + "_" + "state", - size = size, - inputs = Input(name, input_layer_argument = "state"), - ) - -def LstmRecurrentUnitNaive(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if para_prefix is None: + name=name + "_" + "state", + size=size, + inputs=Input( + name, input_layer_argument="state"), ) + + +def LstmRecurrentUnitNaive(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) ExpressionLayer( - name = name + "_" + "input_s", - size = size, - active_type = active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=0)], - ) + name=name + "_" + "input_s", + size=size, + active_type=active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=0) + ], ) ExpressionLayer( - name = name + "_" + "input_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_input_check.w")], - ) + name=name + "_" + "input_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size), DotMulProjection( + state_memory, parameter_name=para_prefix + "_input_check.w") + ], ) ExpressionLayer( - name = name + "_" + "forget_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*2), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_forget_check.w")], - ) + name=name + "_" + "forget_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 2), + DotMulProjection( + state_memory, parameter_name=para_prefix + "_forget_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state", - inputs = [DotMulOperator([name + "_" + "input_s", - name + "_" + "input_gate"]), - DotMulOperator([state_memory, - name + "_" + "forget_gate"]), - ], - ) + name=name + "_" + "state", + inputs=[ + DotMulOperator([name + "_" + "input_s", name + "_" + "input_gate"]), + DotMulOperator([state_memory, name + "_" + "forget_gate"]), + ], ) ExpressionLayer( - name = name + "_" + "output_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*3), - DotMulProjection(name + "_" + "state", - parameter_name = para_prefix + "_output_check.w")], - ) + name=name + "_" + "output_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 3), + DotMulProjection( + name + "_" + "state", + parameter_name=para_prefix + "_output_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state_atv", - active_type = state_active_type, - inputs = IdentityProjection(name + "_" + "state"), - ) + name=name + "_" + "state_atv", + active_type=state_active_type, + inputs=IdentityProjection(name + "_" + "state"), ) ExpressionLayer( - name = name, - inputs = DotMulOperator([name + "_" + "state_atv", - name + "_" + "output_gate"]), - ) + name=name, + inputs=DotMulOperator( + [name + "_" + "state_atv", name + "_" + "output_gate"]), ) + # like LstmRecurrentUnit, but it's a layer group. # it is equivalent to LstmLayer -def LstmRecurrentLayerGroup(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def LstmRecurrentLayerGroup(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 4, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 4, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) LstmRecurrentUnit( - name = name, - size = size, - active_type = active_type, - state_active_type = state_active_type, - gate_active_type = gate_active_type, - inputs = [IdentityProjection(input_layer_name)], - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + state_active_type=state_active_type, + gate_active_type=gate_active_type, + inputs=[IdentityProjection(input_layer_name)], + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - # gated recurrent unit, can be used in recurrent machine # *inputs* should be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], @@ -184,142 +198,157 @@ def LstmRecurrentLayerGroup(name, size, # two GatedRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def GatedRecurrentUnit(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + +def GatedRecurrentUnit(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) GruStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_gate.b"), - inputs = [input_layer_name, - Input(out_memory, parameter_name = para_prefix + "_gate.w")], - active_type = active_type, - active_gate_type = gate_active_type, - ) - -def GatedRecurrentUnitNaive(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_gate.b"), + inputs=[ + input_layer_name, Input( + out_memory, parameter_name=para_prefix + "_gate.w") + ], + active_type=active_type, + active_gate_type=gate_active_type, ) + + +def GatedRecurrentUnitNaive(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) Layer( - name = name + "_" + "update_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_update_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=0), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_update_gate.w")], - ) + name=name + "_" + "update_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_update_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=0), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_update_gate.w") + ], ) Layer( - name = name + "_" + "reset_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_reset_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_reset_gate.w")], - ) + name=name + "_" + "reset_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_reset_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_reset_gate.w") + ], ) ExpressionLayer( - name = name + "_" + "reset_output", - inputs = DotMulOperator([out_memory, name + "_" + "reset_gate"]), - ) + name=name + "_" + "reset_output", + inputs=DotMulOperator([out_memory, name + "_" + "reset_gate"]), ) Layer( - name = name + "_" + "output_candidate", - type = "mixed", - size = size, - active_type = active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_output_candidate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size*2), - FullMatrixProjection(name + "_" + "reset_output", - parameter_name = para_prefix + "_output_candidate.w")], - ) - ExpressionLayer( #element-wise interpolation - name = name, - inputs = [IdentityProjection(out_memory), - DotMulOperator([out_memory, - name + "_" + "update_gate"], scale=-1.0), - DotMulOperator([name + "_" + "output_candidate", - name + "_" + "update_gate"]), - ], - ) + name=name + "_" + "output_candidate", + type="mixed", + size=size, + active_type=active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_output_candidate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size * 2), FullMatrixProjection( + name + "_" + "reset_output", + parameter_name=para_prefix + "_output_candidate.w") + ], ) + ExpressionLayer( #element-wise interpolation + name=name, + inputs=[ + IdentityProjection(out_memory), + DotMulOperator( + [out_memory, name + "_" + "update_gate"], scale=-1.0), + DotMulOperator( + [name + "_" + "output_candidate", name + "_" + "update_gate"]), + ], ) + # like GatedRecurrentUnit, but it's a layer group. # it is equivalent to GatedRecurrentLayer. -def GatedRecurrentLayerGroup(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def GatedRecurrentLayerGroup(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) GatedRecurrentUnit( - name = name, - size = size, - active_type = active_type, - gate_active_type = gate_active_type, - inputs = input_layer_name, #transform outside - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + gate_active_type=gate_active_type, + inputs=input_layer_name, #transform outside + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - diff --git a/python/paddle/trainer_config_helpers/__init__.py b/python/paddle/trainer_config_helpers/__init__.py index 451b9ac3396ea..adebebba2523f 100644 --- a/python/paddle/trainer_config_helpers/__init__.py +++ b/python/paddle/trainer_config_helpers/__init__.py @@ -20,3 +20,6 @@ from networks import * from optimizers import * from attrs import * + +# This will enable operator overload for LayerOutput +import math diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 292014519374e..6261934e1bc8e 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -12,20 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ["TanhActivation", "SigmoidActivation", - "SoftmaxActivation", "IdentityActivation", "LinearActivation", - 'SequenceSoftmaxActivation', 'ExpActivation', - "ReluActivation", "BReluActivation", "SoftReluActivation", - "STanhActivation", - "AbsActivation", "SquareActivation", - "BaseActivation"] +__all__ = [ + "TanhActivation", "SigmoidActivation", "SoftmaxActivation", + "IdentityActivation", "LinearActivation", 'SequenceSoftmaxActivation', + 'ExpActivation', "ReluActivation", "BReluActivation", "SoftReluActivation", + "STanhActivation", "AbsActivation", "SquareActivation", "BaseActivation" +] class BaseActivation(object): """ - A mark for activation class. + A mark for activation class. Each activation inherit BaseActivation, which has two parameters. - + :param name: activation name in paddle config. :type name: basestring :param support_hppl: True if supported by hppl. HPPL is a library used by paddle @@ -51,7 +50,8 @@ class TanhActivation(BaseActivation): f(z)=tanh(z)=\\frac{e^z-e^{-z}}{e^z+e^{-z}} """ - def __init__(self): BaseActivation.__init__(self, 'tanh', True) + def __init__(self): + BaseActivation.__init__(self, 'tanh', True) class SigmoidActivation(BaseActivation): @@ -63,7 +63,8 @@ class SigmoidActivation(BaseActivation): f(z) = \\frac{1}{1+exp(-z)} """ - def __init__(self): BaseActivation.__init__(self, 'sigmoid', True) + def __init__(self): + BaseActivation.__init__(self, 'sigmoid', True) class SoftmaxActivation(BaseActivation): @@ -104,7 +105,8 @@ class IdentityActivation(BaseActivation): Just do nothing for output both forward/backward. """ - def __init__(self): BaseActivation.__init__(self, '', False) + def __init__(self): + BaseActivation.__init__(self, '', False) LinearActivation = IdentityActivation @@ -124,7 +126,8 @@ class ReluActivation(BaseActivation): 0 &\\quad\\mathrm{otherwize} """ - def __init__(self): BaseActivation.__init__(self, 'relu', True) + def __init__(self): + BaseActivation.__init__(self, 'relu', True) class BReluActivation(BaseActivation): @@ -141,7 +144,8 @@ class BReluActivation(BaseActivation): 0 &\\quad \\mathrm{otherwise} """ - def __init__(self): BaseActivation.__init__(self, 'brelu', False) + def __init__(self): + BaseActivation.__init__(self, 'brelu', False) class SoftReluActivation(BaseActivation): @@ -149,7 +153,9 @@ class SoftReluActivation(BaseActivation): SoftRelu Activation. """ - def __init__(self): BaseActivation.__init__(self, 'softrelu', False) + def __init__(self): + BaseActivation.__init__(self, 'softrelu', False) + class STanhActivation(BaseActivation): """ @@ -160,7 +166,8 @@ class STanhActivation(BaseActivation): f(z) = 1.7159 * tanh(2/3*z) """ - def __init__(self): BaseActivation.__init__(self, 'stanh', False) + def __init__(self): + BaseActivation.__init__(self, 'stanh', False) class AbsActivation(BaseActivation): @@ -178,7 +185,8 @@ class AbsActivation(BaseActivation): 0 &\\quad if \\quad z = 0 """ - def __init__(self): BaseActivation.__init__(self, 'abs', False) + def __init__(self): + BaseActivation.__init__(self, 'abs', False) class SquareActivation(BaseActivation): @@ -189,13 +197,29 @@ class SquareActivation(BaseActivation): f(z) = z^2. """ - def __init__(self): BaseActivation.__init__(self, 'square', False) + def __init__(self): + BaseActivation.__init__(self, 'square', False) + class ExpActivation(BaseActivation): """ Exponential Activation. - + .. math:: f(z) = e^z. """ - def __init__(self): BaseActivation.__init__(self, 'exponential', False) + + def __init__(self): + BaseActivation.__init__(self, 'exponential', False) + + +class LogActivation(BaseActivation): + """ + Logarithm Activation. + + .. math:: + f(z) = log(z) + """ + + def __init__(self): + BaseActivation.__init__(self, 'log', False) diff --git a/python/paddle/trainer_config_helpers/attrs.py b/python/paddle/trainer_config_helpers/attrs.py index d263441247332..54169f382f164 100644 --- a/python/paddle/trainer_config_helpers/attrs.py +++ b/python/paddle/trainer_config_helpers/attrs.py @@ -13,8 +13,9 @@ # limitations under the License. from paddle.trainer.config_parser import * -__all__ = ['ParamAttr', 'ExtraAttr', 'ParameterAttribute', - 'ExtraLayerAttribute'] +__all__ = [ + 'ParamAttr', 'ExtraAttr', 'ParameterAttribute', 'ExtraLayerAttribute' +] def convert_and_compare(x, Type): @@ -25,7 +26,8 @@ def convert_and_compare(x, Type): :param Type: target type to check x over """ - return type(x)(Type(x))==x + return type(x)(Type(x)) == x + def is_compatible_with(x, Type): """ @@ -38,9 +40,9 @@ def is_compatible_with(x, Type): return True try: if float == Type or int == Type: - # avoid those types that can be converted to float/int but not very - # meaningful and could potentially lead to error - # i.e., str and bool typed value should not be used for initializing float/int variable + # avoid those types that can be converted to float/int but not very + # meaningful and could potentially lead to error + # i.e., str and bool typed value should not be used for initializing float/int variable if not isinstance(x, str) and not isinstance(x, bool): return convert_and_compare(x, Type) elif bool == Type: @@ -91,9 +93,17 @@ class ParameterAttribute(object): :type sparse_update: bool """ - def __init__(self, name=None, is_static=False, initial_std=None, - initial_mean=None, initial_max=None, initial_min=None, - l1_rate=None, l2_rate=None, learning_rate=None, momentum=None, + def __init__(self, + name=None, + is_static=False, + initial_std=None, + initial_mean=None, + initial_max=None, + initial_min=None, + l1_rate=None, + l2_rate=None, + learning_rate=None, + momentum=None, sparse_update=False): # initialize strategy. if is_static: @@ -183,7 +193,10 @@ class ExtraLayerAttribute(object): :type device: int """ - def __init__(self, error_clipping_threshold=None, drop_rate=None, device=None): + def __init__(self, + error_clipping_threshold=None, + drop_rate=None, + device=None): self.attr = dict() if isinstance(error_clipping_threshold, float): assert error_clipping_threshold > 0 @@ -200,8 +213,8 @@ def check(self, layer_name): for key in self.attr: if not hasattr(self, 'can_%s' % key) or \ not getattr(self, 'can_%s' % key): - raise NotImplementedError( - "Layer %s cannot support %s" % (layer_name, key)) + raise NotImplementedError("Layer %s cannot support %s" % + (layer_name, key)) @staticmethod def to_kwargs(attr): diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index 8ada3903dc06b..b41097953dad8 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Data Sources are helpers to define paddle training data or testing data. """ @@ -26,8 +25,12 @@ __all__ = ['define_py_data_sources2'] -def define_py_data_source(file_list, cls, module, - obj, args=None, async=False, +def define_py_data_source(file_list, + cls, + module, + obj, + args=None, + async=False, data_cls=PyData): """ Define a python data source. @@ -68,7 +71,7 @@ def define_py_data_source(file_list, cls, module, file_list_name = 'train.list' if isinstance(cls, TestData): file_list_name = 'test.list' - with open(file_list_name, 'r') as f: + with open(file_list_name, 'w') as f: f.writelines(file_list) file_list = file_list_name @@ -76,26 +79,36 @@ def define_py_data_source(file_list, cls, module, args = pickle.dumps(args, 0) if data_cls is None: + def py_data2(files, load_data_module, load_data_object, load_data_args, - **kwargs): + **kwargs): data = DataBase() data.type = 'py2' data.files = files data.load_data_module = load_data_module data.load_data_object = load_data_object data.load_data_args = load_data_args + data.async_load_data = True return data - data_cls = py_data2 - - cls(data_cls(files=file_list, - load_data_module=module, - load_data_object=obj, - load_data_args=args, - async_load_data=async)) + data_cls = py_data2 -def define_py_data_sources(train_list, test_list, module, obj, args=None, - train_async=False, data_cls=PyData): + cls( + data_cls( + files=file_list, + load_data_module=module, + load_data_object=obj, + load_data_args=args, + async_load_data=async)) + + +def define_py_data_sources(train_list, + test_list, + module, + obj, + args=None, + train_async=False, + data_cls=PyData): """ The annotation is almost the same as define_py_data_sources2, except that it can specific train_async and data_cls. @@ -124,8 +137,8 @@ def define_py_data_sources(train_list, test_list, module, obj, args=None, """ def __is_splitable__(o): - return (isinstance(o, list) or isinstance(o, tuple) - ) and hasattr(o, '__len__') and len(o) == 2 + return (isinstance(o, list) or + isinstance(o, tuple)) and hasattr(o, '__len__') and len(o) == 2 assert train_list is not None or test_list is not None assert module is not None and obj is not None @@ -138,7 +151,7 @@ def __is_splitable__(o): test_obj = obj train_obj = obj if __is_splitable__(obj): - train_module, test_module = module + train_obj, test_obj = obj if args is None: args = "" @@ -195,9 +208,10 @@ def define_py_data_sources2(train_list, test_list, module, obj, args=None): :return: None :rtype: None """ - define_py_data_sources(train_list=train_list, - test_list=test_list, - module=module, - obj=obj, - args=args, - data_cls=None) + define_py_data_sources( + train_list=train_list, + test_list=test_list, + module=module, + obj=obj, + args=args, + data_cls=None) diff --git a/python/paddle/trainer_config_helpers/default_decorators.py b/python/paddle/trainer_config_helpers/default_decorators.py index b20aebc685fe5..c01050e338d59 100644 --- a/python/paddle/trainer_config_helpers/default_decorators.py +++ b/python/paddle/trainer_config_helpers/default_decorators.py @@ -13,20 +13,23 @@ # limitations under the License. import functools +import inspect from .attrs import ParamAttr from .activations import TanhActivation from paddle.trainer.config_parser import * -__all__ = ['wrap_name_default', 'wrap_param_attr_default', - 'wrap_bias_attr_default', 'wrap_act_default', - 'wrap_param_default'] +__all__ = [ + 'wrap_name_default', 'wrap_param_attr_default', 'wrap_bias_attr_default', + 'wrap_act_default', 'wrap_param_default' +] def __default_not_set_callback__(kwargs, name): return name not in kwargs or kwargs[name] is None -def wrap_param_default(param_names=None, default_factory=None, +def wrap_param_default(param_names=None, + default_factory=None, not_set_callback=__default_not_set_callback__): assert param_names is not None assert isinstance(param_names, list) or isinstance(param_names, tuple) @@ -37,8 +40,13 @@ def __impl__(func): @functools.wraps(func) def __wrapper__(*args, **kwargs): if len(args) != 0: - logger.warning("please use keyword arguments in paddle config.") - + argspec = inspect.getargspec(func) + num_positional = len(argspec.args) + if argspec.defaults: + num_positional -= len(argspec.defaults) + if not argspec.varargs and len(args) > num_positional: + logger.fatal( + "Must use keyword arguments for non-positional args") for name in param_names: if not_set_callback(kwargs, name): # Not set kwargs[name] = default_factory(func) @@ -107,13 +115,13 @@ def wrap_param_attr_default(param_names=None, default_factory=None): return wrap_param_default(param_names, default_factory) -def wrap_bias_attr_default(param_names=None, default_factory=None, +def wrap_bias_attr_default(param_names=None, + default_factory=None, has_bias=True): if param_names is None: param_names = ['bias_attr'] if default_factory is None: - default_factory = lambda _: ParamAttr(initial_std=0., - initial_mean=0.) + default_factory = lambda _: ParamAttr(initial_std=0., initial_mean=0.) def __bias_attr_not_set__(kwargs, name): if has_bias: diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index ded124a5c8ca4..dc6a36392f9c6 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -15,13 +15,14 @@ from paddle.trainer.config_parser import * from default_decorators import * -__all__ = ["evaluator_base","classification_error_evaluator", "auc_evaluator", - "pnpair_evaluator", "precision_recall_evaluator", - "ctc_error_evaluator", "chunk_evaluator", "sum_evaluator", - "column_sum_evaluator", "value_printer_evaluator", - "gradient_printer_evaluator", "maxid_printer_evaluator", - "maxframe_printer_evaluator", "seqtext_printer_evaluator", - "classification_error_printer_evaluator"] +__all__ = [ + "evaluator_base", "classification_error_evaluator", "auc_evaluator", + "pnpair_evaluator", "precision_recall_evaluator", "ctc_error_evaluator", + "chunk_evaluator", "sum_evaluator", "column_sum_evaluator", + "value_printer_evaluator", "gradient_printer_evaluator", + "maxid_printer_evaluator", "maxframe_printer_evaluator", + "seqtext_printer_evaluator", "classification_error_printer_evaluator" +] class EvaluatorAttribute(object): @@ -32,10 +33,7 @@ class EvaluatorAttribute(object): FOR_UTILS = 1 << 4 KEYS = [ - "for_classification", - "for_regression", - "for_rank", - "for_print", + "for_classification", "for_regression", "for_rank", "for_print", "for_utils" ] @@ -55,22 +53,23 @@ def impl(method): setattr(method, EvaluatorAttribute.to_key(attr), True) method.is_evaluator = True return method + return impl -def evaluator_base( - input, - type, - label=None, - weight=None, - name=None, - chunk_scheme=None, - num_chunk_types=None, - classification_threshold=None, - positive_label=None, - dict_file=None, - result_file=None, - num_results=None, - delimited=None): + +def evaluator_base(input, + type, + label=None, + weight=None, + name=None, + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None): """ Evaluator will evaluate the network status while training/testing. @@ -130,14 +129,14 @@ def evaluator_base( result_file=result_file, delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() -def classification_error_evaluator( - input, - label, - name=None, - weight=None, - threshold=None): +def classification_error_evaluator(input, + label, + name=None, + weight=None, + threshold=None): """ Classification Error Evaluator. It will print error rate for classification. @@ -170,13 +169,14 @@ def classification_error_evaluator( :return: None. """ - evaluator_base(name=name, - type="classification_error", - input=input, - label=label, - weight=weight, - classification_threshold=threshold, - ) + evaluator_base( + name=name, + type="classification_error", + input=input, + label=label, + weight=weight, + classification_threshold=threshold, ) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -184,8 +184,7 @@ def auc_evaluator( input, label, name=None, - weight=None, - ): + weight=None, ): """ Auc Evaluator which adapts to binary classification. @@ -205,11 +204,13 @@ def auc_evaluator( [sample_num, 1]. :type weight: LayerOutput """ - evaluator_base(name=name, - type="last-column-auc", - input=input, - label=label, - weight=weight) + evaluator_base( + name=name, + type="last-column-auc", + input=input, + label=label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_RANK) @wrap_name_default() @@ -218,8 +219,7 @@ def pnpair_evaluator( label, info, name=None, - weight=None, - ): + weight=None, ): """ Positive-negative pair rate Evaluator which adapts to rank task like learning to rank. This evaluator must contain at least three layers. @@ -242,12 +242,14 @@ def pnpair_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="pnpair", - input=input, - label=label, - info=info, - weight=weight) + evaluator_base( + name=name, + type="pnpair", + input=input, + label=label, + info=info, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -256,8 +258,7 @@ def precision_recall_evaluator( label, positive_label=None, weight=None, - name=None, - ): + name=None, ): """ An Evaluator to calculate precision and recall, F1-score. It is adapt to the task with multiple labels. @@ -286,20 +287,21 @@ def precision_recall_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="precision_recall", - input=input, - label=label, - positive_label=positive_label, - weight=weight) + evaluator_base( + name=name, + type="precision_recall", + input=input, + label=label, + positive_label=positive_label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() def ctc_error_evaluator( input, label, - name=None, - ): + name=None, ): """ This evaluator is to calculate sequence-to-sequence edit distance. @@ -317,10 +319,9 @@ def ctc_error_evaluator( label for ctc_layer :type label: LayerOutput """ - evaluator_base(name=name, - type="ctc_edit_distance", - input=input, - label=label) + evaluator_base( + name=name, type="ctc_edit_distance", input=input, label=label) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -328,8 +329,7 @@ def chunk_evaluator( input, name=None, chunk_scheme=None, - num_chunk_types=None, - ): + num_chunk_types=None, ): """ Chunk evaluator is used to evaluate segment labelling accuracy for a sequence. It calculates the chunk detection F1 score. @@ -375,19 +375,20 @@ def chunk_evaluator( :type chunk_scheme: basestring :param num_chunk_types: number of chunk types other than "other" """ - evaluator_base(name=name, - type="chunk", - input=input, - chunk_scheme=chunk_scheme, - num_chunk_types=num_chunk_types) + evaluator_base( + name=name, + type="chunk", + input=input, + chunk_scheme=chunk_scheme, + num_chunk_types=num_chunk_types) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ An Evaluator to sum the result of input. @@ -405,18 +406,15 @@ def sum_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="sum", - input=input, - weight=weight) + evaluator_base(name=name, type="sum", input=input, weight=weight) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def column_sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ This Evaluator is used to sum the last column of input. @@ -431,22 +429,22 @@ def column_sum_evaluator( :param input: Input Layer name. :type input: LayerOutput """ - evaluator_base(name=name, - type="last-column-sum", - input=input, - weight=weight) + evaluator_base( + name=name, type="last-column-sum", input=input, weight=weight) + """ The following are printer Evaluators which are usually used to print the result, like value or gradient of input layers, the results generated in machine translation, the classification error etc. """ + + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def value_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the values of input layers. It contains one or more input layers. @@ -462,16 +460,14 @@ def value_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="value_printer", - input=input) + evaluator_base(name=name, type="value_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def gradient_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the gradient of input layers. It contains one or more input layers. @@ -487,17 +483,15 @@ def gradient_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="gradient_printer", - input=input) + evaluator_base(name=name, type="gradient_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxid_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print maximum top k values and their indexes of each row of input layers. It contains one or more input layers. @@ -517,18 +511,16 @@ def maxid_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_id_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, type="max_id_printer", input=input, num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxframe_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print the top k frames of each input layers. The input layers should contain sequences info or sequences type. @@ -549,10 +541,12 @@ def maxframe_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_frame_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, + type="max_frame_printer", + input=input, + num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -562,8 +556,7 @@ def seqtext_printer_evaluator( id_input=None, dict_file=None, delimited=None, - name=None, - ): + name=None, ): """ Sequence text printer will print text according to index matrix and a dictionary. There can be multiple input to this layer: @@ -636,12 +629,14 @@ def seqtext_printer_evaluator( inputs = [id_input, input] input.parents.append(id_input) - evaluator_base(name=name, - type="seq_text_printer", - input=inputs, - dict_file=dict_file, - result_file=result_file, - delimited=delimited) + evaluator_base( + name=name, + type="seq_text_printer", + input=inputs, + dict_file=dict_file, + result_file=result_file, + delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -649,8 +644,7 @@ def classification_error_printer_evaluator( input, label, threshold=0.5, - name=None, - ): + name=None, ): """ This Evaluator is used to print the classification error of each sample. @@ -667,8 +661,9 @@ def classification_error_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="classification_error_printer", - input=input, - label=label, - classification_threshold=threshold) + evaluator_base( + name=name, + type="classification_error_printer", + input=input, + label=label, + classification_threshold=threshold) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index c355dc042ac18..b5e10ef81009a 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -29,33 +29,84 @@ import pickle import copy -__all__ = ["full_matrix_projection", "AggregateLevel", "ExpandLevel", - "identity_projection", "dotmul_projection", "dotmul_operator", - "table_projection", "mixed_layer", "data_layer", - "embedding_layer", "fc_layer", "grumemory", - "pooling_layer", "lstmemory", "last_seq", "first_seq", - "cos_sim", "hsigmoid", - "regression_cost", 'classification_cost', "LayerOutput", - 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', - 'img_cmrnorm_layer', 'addto_layer', - 'concat_layer', 'lstm_step_layer', 'recurrent_group', - 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', - 'power_layer', 'interpolation_layer', 'trans_layer', - 'sum_to_one_norm_layer', - 'get_output_layer', 'LayerType', 'context_projection', - 'beam_search', 'maxid_layer', 'GeneratedInput', 'SubsequenceInput', - 'gru_step_layer', 'recurrent_layer', - 'BaseGeneratedInput', 'conv_operator', 'conv_shift_layer', - 'tensor_layer', 'selective_fc_layer', 'sampling_id_layer', - 'slope_intercept_layer', 'trans_full_matrix_projection', - 'linear_comb_layer', - 'convex_comb_layer', 'ctc_layer', 'crf_layer', 'crf_decoding_layer', - 'cross_entropy_with_selfnorm', 'cross_entropy', - 'multi_binary_label_cross_entropy', - 'rank_cost', 'lambda_cost', 'huber_cost', - # 'block_expand_layer', # TODO(yuyang18): this layer is not correct - 'out_prod_layer', 'print_layer' - ] +__all__ = [ + "full_matrix_projection", + "AggregateLevel", + "ExpandLevel", + "identity_projection", + "dotmul_projection", + "dotmul_operator", + "repeat_layer", + "table_projection", + "mixed_layer", + "data_layer", + "embedding_layer", + "fc_layer", + "grumemory", + "pooling_layer", + "lstmemory", + "last_seq", + "first_seq", + "cos_sim", + "hsigmoid", + "conv_projection", + "regression_cost", + 'classification_cost', + "LayerOutput", + 'img_conv_layer', + 'img_pool_layer', + 'batch_norm_layer', + 'img_cmrnorm_layer', + 'addto_layer', + 'concat_layer', + 'lstm_step_layer', + 'recurrent_group', + 'memory', + 'StaticInput', + 'expand_layer', + 'scaling_layer', + 'scaling_projection', + 'power_layer', + 'interpolation_layer', + 'bilinear_interp_layer', + 'trans_layer', + 'sum_to_one_norm_layer', + 'get_output_layer', + 'LayerType', + 'context_projection', + 'beam_search', + 'maxid_layer', + 'GeneratedInput', + 'SubsequenceInput', + 'gru_step_layer', + 'recurrent_layer', + 'BaseGeneratedInput', + 'conv_operator', + 'conv_shift_layer', + 'tensor_layer', + 'selective_fc_layer', + 'sampling_id_layer', + 'slope_intercept_layer', + 'trans_full_matrix_projection', + 'linear_comb_layer', + 'convex_comb_layer', + 'ctc_layer', + 'crf_layer', + 'crf_decoding_layer', + 'nce_layer', + 'cross_entropy_with_selfnorm', + 'cross_entropy', + 'multi_binary_label_cross_entropy', + 'sum_cost', + 'rank_cost', + 'lambda_cost', + 'huber_cost', + 'block_expand_layer', + 'maxout_layer', + 'out_prod_layer', + 'print_layer', + 'spp_layer', +] class LayerType(object): @@ -77,6 +128,7 @@ class LayerType(object): COSINE_SIM = 'cos' HSIGMOID = 'hsigmoid' CONV_LAYER = "conv" + CONVTRANS_LAYER = "convt" POOL_LAYER = "pool" BATCH_NORM_LAYER = 'batch_norm' NORM_LAYER = 'norm' @@ -92,10 +144,12 @@ class LayerType(object): EXPAND_LAYER = 'expand' INTERPOLATION_LAYER = 'interpolation' + BILINEAR_INTERP_LAYER = 'bilinear_interp' POWER_LAYER = 'power' SCALING_LAYER = 'scaling' TRANS_LAYER = 'trans' OUT_PROD_LAYER = 'out_prod' + FEATURE_MAP_EXPAND_LAYER = 'featmap_expand' MEMORY = 'memory' MAXID_LAYER = 'maxid' @@ -109,12 +163,15 @@ class LayerType(object): SLOPE_INTERCEPT_LAYER = "slope_intercept" LINEAR_COMBINATION_LAYER = "convex_comb" BLOCK_EXPAND = "blockexpand" + MAXOUT = "maxout" + SPP_LAYER = "spp" PRINT_LAYER = "print" CTC_LAYER = "ctc" CRF_LAYER = "crf" CRF_DECODING_LAYER = "crf_decoding" + NCE_LAYER = 'nce' RANK_COST = "rank-cost" LAMBDA_COST = "lambda_cost" @@ -123,6 +180,7 @@ class LayerType(object): CROSS_ENTROPY_WITH_SELFNORM = "multi_class_cross_entropy_with_selfnorm" SOFT_BIN_CLASS_CROSS_ENTROPY = "soft_binary_class_cross_entropy" MULTI_BIN_LABEL_CROSS_ENTROPY = "multi_binary_label_cross_entropy" + SUM_COST = "sum_cost" @staticmethod def is_layer_type(type_name): @@ -168,14 +226,22 @@ class LayerOutput(object): :param activation: Layer Activation. :type activation: BaseActivation. :param parents: Layer's parents. - :type parents: list|tuple|collection.Sequence - """ - - def __init__(self, name, layer_type, parents=None, activation=None, - num_filters=None, img_norm_type=None, size=None, outputs=None, + :type parents: list|tuple|collections.Sequence + """ + + def __init__(self, + name, + layer_type, + parents=None, + activation=None, + num_filters=None, + img_norm_type=None, + size=None, + outputs=None, reverse=None): assert isinstance(name, basestring) assert isinstance(layer_type, basestring) + assert size is not None assert LayerType.is_layer_type(layer_type) self.name = name self.layer_type = layer_type @@ -210,8 +276,9 @@ def __str__(self): def layer_support(*attrs): - attrs_list = list(attrs) + attrs_list = list(attrs) attrs_list.append(DEVICE) + def decorator(method): @functools.wraps(method) def wrapper(*args, **kwargs): @@ -271,9 +338,8 @@ def full_matrix_projection(input, size=0, param_attr=None): :return: A FullMatrixProjection Object. :rtype: FullMatrixProjection """ - proj = FullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = FullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -308,9 +374,8 @@ def trans_full_matrix_projection(input, size=0, param_attr=None): :return: A TransposedFullMatrixProjection Object. :rtype: TransposedFullMatrixProjection """ - proj = TransposedFullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TransposedFullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -354,9 +419,8 @@ def table_projection(input, size=0, param_attr=None): :return: A TableProjection Object. :rtype: TableProjection """ - proj = TableProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TableProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -395,19 +459,47 @@ def identity_projection(input, offset=None): :type input: LayerOutput :param offset: Offset, None if use default. :type offset: int - :return: A IdentityProjection or IdentityOffsetProjection Object + :return: A IdentityProjection or IdentityOffsetProjection object :rtype: IdentityProjection or IdentityOffsetProjection """ if offset is None: proj = IdentityProjection(input_layer_name=input.name) proj.origin = input else: - proj = IdentityOffsetProjection(input_layer_name=input.name, - offset=offset) + proj = IdentityOffsetProjection( + input_layer_name=input.name, offset=offset) proj.origin = input return proj +@wrap_param_attr_default() +def scaling_projection(input, param_attr=None): + """ + scaling_projection multiplies the input with a scalar parameter and add to + the output. + + .. math:: + out += w * in + + The example usage is: + + .. code-block:: python + + proj = scaling_projection(input=layer) + + :param input: Input Layer. + :type input: LayerOutput + :param param_attr: Parameter config, None if use default. + :type param_attr: ParameterAttribute + :return: A ScalingProjection object + :rtype: ScalingProjection + """ + proj = ScalingProjection(input_layer_name=input.name, + **param_attr.attr) + proj.origin = input + return proj + + @wrap_param_attr_default() def dotmul_projection(input, param_attr=None): """ @@ -432,9 +524,8 @@ def dotmul_projection(input, param_attr=None): :return: A DotMulProjection Object. :rtype: DotMulProjection """ - proj = DotMulProjection(input_layer_name=input.name, - size=input.size, - **param_attr.attr) + proj = DotMulProjection( + input_layer_name=input.name, size=input.size, **param_attr.attr) proj.origin = input return proj @@ -467,21 +558,22 @@ def dotmul_operator(a=None, b=None, scale=1, **kwargs): if 'x' in kwargs or 'y' in kwargs: logger.warning('x and y arguments for dotmul_operator is deprecated. ' 'Please use a and b as parameter.') - a = kwargs.get('x', a) # For Backward capacity. + a = kwargs.get('x', a) # For Backward capacity. b = kwargs.get('y', b) assert isinstance(a, LayerOutput) assert isinstance(b, LayerOutput) if a.size is not None and b.size is not None: assert a.size == b.size - op = DotMulOperator(input_layer_names=[a.name, b.name], - scale=scale) + op = DotMulOperator(input_layer_names=[a.name, b.name], scale=scale) op.origin = [a, b] return op @wrap_bias_attr_default(['padding_attr']) -def context_projection(input, context_len, context_start=None, +def context_projection(input, + context_len, + context_start=None, padding_attr=False): """ Context Projection. @@ -518,11 +610,12 @@ def context_projection(input, context_len, context_start=None, if trainable: extra_dict = padding_attr.attr - proj = ContextProjection(input_layer_name=input.name, - context_length=context_len, - context_start=context_start, - trainable_padding=trainable, - **extra_dict) + proj = ContextProjection( + input_layer_name=input.name, + context_length=context_len, + context_start=context_start, + trainable_padding=trainable, + **extra_dict) proj.origin = input return proj @@ -536,8 +629,7 @@ class AddToSealedMixedLayerException(Exception): def __init__(self): Exception.__init__(self) - def __init__(self, name, size, act, bias_attr, layer_attr, - parents=None): + def __init__(self, name, size, act, bias_attr, layer_attr, parents=None): """ Ctor. :param name: layer name. @@ -554,14 +646,19 @@ def __init__(self, name, size, act, bias_attr, layer_attr, :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None """ - LayerOutput.__init__(self, name, LayerType.MIXED_LAYER, parents, - size=size, activation=act) + LayerOutput.__init__( + self, + name, + LayerType.MIXED_LAYER, + parents, + size=size, + activation=act) self.bias_attr = bias_attr self.layer_attr = layer_attr self.inputs = [] self.finalized = False - def __add__(self, other): + def __iadd__(self, other): """ + += operator :param other: Other projection. @@ -587,21 +684,27 @@ def __enter__(self): def __exit__(self, *args, **kwargs): del args, kwargs # unused parameter to suppress warning assert len(self.inputs) != 0 - MixedLayer( + ml = MixedLayer( name=self.name, size=self.size, active_type=self.activation.name, bias=ParamAttr.to_bias(self.bias_attr), inputs=self.inputs, - **ExtraLayerAttribute.to_kwargs(self.layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(self.layer_attr)) + # update the size which might be computed inside MixedLayer + # according to the operator's output size + self.size = ml.config.size @wrap_name_default("mixed") @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(ERROR_CLIPPING, DROPOUT) -def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, +def mixed_layer(size=0, + input=None, + name=None, + act=None, + bias_attr=False, layer_attr=None): """ Mixed Layer. A mixed layer will add all inputs together, then activate. @@ -646,8 +749,12 @@ def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, if input is None: return MixedLayerType(name, size, act, bias_attr, layer_attr) else: - with mixed_layer(name=name, size=size, act=act, bias_attr=bias_attr, - layer_attr=layer_attr) as m: + with mixed_layer( + name=name, + size=size, + act=act, + bias_attr=bias_attr, + layer_attr=layer_attr) as m: if isinstance(input, collections.Sequence): for each in input: m += each @@ -677,8 +784,11 @@ def data_layer(name, size, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer(type=LayerType.DATA, name=name, size=size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + type=LayerType.DATA, + name=name, + size=size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.DATA, size=size) @@ -704,9 +814,12 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - with mixed_layer(name=name, size=size, act=LinearActivation(), - bias_attr=False, - layer_attr=layer_attr) as mix: + with mixed_layer( + name=name, + size=size, + act=LinearActivation(), + bias_attr=False, + layer_attr=layer_attr) as mix: mix += table_projection(input=input, size=size, param_attr=param_attr) return mix @@ -716,8 +829,13 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): @wrap_bias_attr_default() @wrap_act_default() @layer_support(ERROR_CLIPPING, DROPOUT) -def fc_layer(input, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def fc_layer(input, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ Helper for declare fully connected layer. @@ -769,17 +887,17 @@ def fc_layer(input, size, act=None, name=None, assert isinstance(input, collections.Sequence) Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ], name=name, type=LayerType.FC_LAYER, size=size, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.FC_LAYER, input, activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.FC_LAYER, input, activation=act, size=size) @wrap_name_default("print") @@ -802,8 +920,7 @@ def print_layer(input, name=None): Layer( name=name, type=LayerType.PRINT_LAYER, - inputs=[l.name for l in input], - ) + inputs=[l.name for l in input], ) # this layer don't return anything, can not be input of other layer. @@ -811,7 +928,10 @@ def print_layer(input, name=None): @wrap_bias_attr_default(has_bias=False) @wrap_param_default(['pooling_type'], default_factory=lambda _: MaxPooling()) @layer_support() -def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, +def pooling_layer(input, + pooling_type=None, + name=None, + bias_attr=None, agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ @@ -858,23 +978,27 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, inputs=[Input(input.name)], bias=ParamAttr.to_bias(bias_attr), trans_type=agg_level, - **extra_dict - ) + **extra_dict) - return LayerOutput(name, pooling_type.name, parents=[input], - size=input.size) + return LayerOutput( + name, pooling_type.name, parents=[input], size=input.size) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act", 'state_act'], act=TanhActivation()) @wrap_name_default("lstmemory") @layer_support(DROPOUT) -def lstmemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - state_act=None, bias_attr=None, param_attr=None, +def lstmemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + state_act=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Long Short-term Memory Cell. @@ -949,30 +1073,38 @@ def lstmemory(input, name=None, reverse=False, act=None, "layer. The lstm size should be equal with input layer size/4. The" " size which is set explicitly will be ignored." % name) - Layer(name=name, - type=LayerType.LSTMEMORY, - active_type=act.name, - active_state_type=state_act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.LSTMEMORY, + active_type=act.name, + active_state_type=state_act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.LSTMEMORY, [input], size=input.size / 4, - reverse=reverse) + return LayerOutput( + name, + LayerType.LSTMEMORY, [input], + size=input.size / 4, + reverse=reverse) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act"], act=TanhActivation()) @wrap_name_default("gru") @layer_support(DROPOUT) -def grumemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - bias_attr=None, param_attr=None, +def grumemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Gate Recurrent Unit Layer. @@ -1063,23 +1195,28 @@ def grumemory(input, name=None, reverse=False, act=None, " and should be input size / 3. Set size explicitly will be " "ignored.") - Layer(name=name, - type=LayerType.GRUMEMORY, - active_type=act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + Layer( + name=name, + type=LayerType.GRUMEMORY, + active_type=act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.GRUMEMORY, [input], size=input.size / 3, - reverse=reverse) + return LayerOutput( + name, + LayerType.GRUMEMORY, [input], + size=input.size / 3, + reverse=reverse) @wrap_name_default() @layer_support() -def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def last_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get Last Timestamp Activation of a sequence. @@ -1105,15 +1242,19 @@ def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_LAST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_LAST_INSTANCE, parents=[input], - size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_LAST_INSTANCE, + parents=[input], + size=input.size) @wrap_name_default() @layer_support() -def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def first_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get First Timestamp Activation of a sequence. @@ -1140,10 +1281,12 @@ def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_FIRST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_FIRST_INSTANCE, - parents=[input], size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_FIRST_INSTANCE, + parents=[input], + size=input.size) class ExpandLevel(object): @@ -1153,7 +1296,8 @@ class ExpandLevel(object): @wrap_name_default() @layer_support() -def expand_layer(input, expand_as, +def expand_layer(input, + expand_as, name=None, bias_attr=False, expand_level=ExpandLevel.FROM_TIMESTEP, @@ -1193,12 +1337,53 @@ def expand_layer(input, expand_as, bias=ParamAttr.to_bias(bias_attr=bias_attr), type=LayerType.EXPAND_LAYER, trans_type=expand_level, - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name=name, - size=input.size, - layer_type=LayerType.EXPAND_LAYER, - parents=[input, expand_as]) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=input.size, + layer_type=LayerType.EXPAND_LAYER, + parents=[input, expand_as]) + + +@wrap_name_default() +@layer_support() +def repeat_layer(input, num_repeats, name=None, layer_attr=None): + """ + A layer for repeating the input for num_repeats times. This is equivalent + to apply concat_layer() with num_repeats same input. + + .. math:: + y = [x, x, \cdots, x] + + The example usage is: + + .. code-block:: python + + expand = repeat_layer(layer, 4) + + :param input: Input layer + :type input: LayerOutput + :param num_repeats: Repeat the input so many times + :type num_repeats: int + :param name: Layer name. + :type name: basestring + :param layer_attr: extra layer attributes. + :type layer_attr: ExtraLayerAttribute. + :return: LayerOutput object. + :rtype: LayerOutput + """ + + l = Layer( + inputs=[input.name], + name=name, + num_filters=num_repeats, + type=LayerType.FEATURE_MAP_EXPAND_LAYER, + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=l.config.size, + layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER, + parents=[input]) @wrap_name_default() @@ -1245,11 +1430,66 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.INTERPOLATION_LAYER, inputs=[weight.name, input[0].name, input[1].name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.INTERPOLATION_LAYER, - parents=[weight, input[0], input[1]], - size=input[0].size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.INTERPOLATION_LAYER, + parents=[weight, input[0], input[1]], + size=input[0].size) + + +@wrap_name_default() +@layer_support() +def bilinear_interp_layer(input, + out_size_x=None, + out_size_y=None, + name=None, + layer_attr=None): + """ + This layer is to implement bilinear interpolation on conv layer output. + + Please refer to Wikipedia: https://en.wikipedia.org/wiki/Bilinear_interpolation + + The simple usage is: + + .. code-block:: python + + bilinear = bilinear_interp_layer(input=layer1, out_size_x=64, out_size_y=64) + + :param input: A input layer. + :type input: LayerOutput. + :param out_size_x: bilinear interpolation output width. + :type out_size_x: int|None + :param out_size_y: bilinear interpolation output height. + :type out_size_y: int|None + :param name: The layer's name, which cna not be specified. + :type name: None|basestring + :param layer_attr: Extra Layer attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + assert input.layer_type == LayerType.CONV_LAYER + assert isinstance(input.activation, LinearActivation) + assert out_size_x > 0 and out_size_y > 0 + assert input.num_filters is not None + num_channels = input.num_filters + l = Layer( + name=name, + inputs=Input( + input.name, + bilinear_interp=BilinearInterp( + out_size_x=out_size_x, + out_size_y=out_size_y, + num_channels=num_channels)), + type=LayerType.BILINEAR_INTERP_LAYER, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.BILINEAR_INTERP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) @wrap_name_default() @@ -1289,10 +1529,9 @@ def power_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.POWER_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POWER_LAYER, - parents=[input, weight], size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.POWER_LAYER, parents=[input, weight], size=input.size) @wrap_name_default() @@ -1334,10 +1573,9 @@ def scaling_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.SCALING_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SCALING_LAYER, parents=[weight, input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SCALING_LAYER, parents=[weight, input], size=input.size) @wrap_name_default() @@ -1370,10 +1608,9 @@ def trans_layer(input, name=None, layer_attr=None): name=name, type=LayerType.TRANS_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TRANS_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TRANS_LAYER, parents=[input], size=input.size) @wrap_name_default() @@ -1415,8 +1652,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): type=LayerType.COSINE_SIM, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) else: if a.size is not None and b.size is not None: assert size == b.size / a.size @@ -1426,17 +1662,21 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): size=size, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b]) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b], size=size) @wrap_name_default() @wrap_bias_attr_default(has_bias=True) @wrap_param_attr_default() @layer_support() -def hsigmoid(input, label, num_classes, name=None, bias_attr=None, - param_attr=None, layer_attr=None): +def hsigmoid(input, + label, + num_classes, + name=None, + bias_attr=None, + param_attr=None, + layer_attr=None): """ Organize the classes into a binary tree. At each node, a sigmoid function is used to calculate the probability of belonging to the right branch. @@ -1491,15 +1731,15 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, ipts_for_layer.append(label.name) parents.append(label) - Layer( + l = Layer( name=name, type=LayerType.HSIGMOID, num_classes=num_classes, bias=ParamAttr.to_bias(bias_attr), inputs=ipts_for_layer, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.HSIGMOID, parents=parents) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.HSIGMOID, parents=parents, size=l.config.size) @wrap_name_default("conv") @@ -1507,11 +1747,23 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, @wrap_bias_attr_default() @wrap_act_default(act=ReluActivation()) @layer_support(DROPOUT) -def img_conv_layer(input, filter_size, num_filters, - name=None, num_channels=None, - act=None, groups=1, stride=1, padding=0, bias_attr=None, - param_attr=None, shared_biases=True, layer_attr=None, - filter_size_y=None, stride_y=None, padding_y=None): +def img_conv_layer(input, + filter_size, + num_filters, + name=None, + num_channels=None, + act=None, + groups=1, + stride=1, + padding=0, + bias_attr=None, + param_attr=None, + shared_biases=True, + layer_attr=None, + filter_size_y=None, + stride_y=None, + padding_y=None, + trans=False): """ Convolution layer for image. Paddle only support square input currently and thus input image's width equals height. @@ -1520,6 +1772,13 @@ def img_conv_layer(input, filter_size, num_filters, `_ . + Convolution Transpose (deconv) layer for image. Paddle only support square + input currently and thus input image's width equals height. + + The details of convolution transpose layer, + please refer to the following explanation and references therein + `_ . The num_channel means input image's channel number. It may be 1 or 3 when input is raw pixels of image(mono or RGB), or it may be the previous layer's num_filters * num_group. @@ -1569,6 +1828,8 @@ def img_conv_layer(input, filter_size, num_filters, :type shared_biases: bool :param layer_attr: Layer Extra Attribute. :type layer_attr: ExtraLayerAttribute + :param trans: true if it is a convTransLayer, false if it is a convLayer + :type trans: bool :return: LayerOutput object. :rtype: LayerOutput """ @@ -1599,35 +1860,57 @@ def img_conv_layer(input, filter_size, num_filters, if param_attr.attr.get('initial_smart'): # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - Layer( + + lt = LayerType.CONVTRANS_LAYER if trans else LayerType.CONV_LAYER + + l = Layer( name=name, - inputs=Input(input.name, conv=Conv( - filter_size=filter_size, padding=padding, stride=stride, - channels=num_channels, groups=groups, - filter_size_y=filter_size_y, padding_y=padding_y, - stride_y=stride_y), - **param_attr.attr), + inputs=Input( + input.name, + conv=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + groups=groups, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y), + **param_attr.attr), active_type=act.name, num_filters=num_filters, bias=ParamAttr.to_bias(bias_attr), shared_biases=shared_biases, - type=LayerType.CONV_LAYER, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.CONV_LAYER, parents=[input], - activation=act, num_filters=num_filters) + type=lt, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + lt, + parents=[input], + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_name_default("pool") @layer_support() -def img_pool_layer(input, pool_size, name=None, - num_channels=None, pool_type=None, - stride=1, start=None, padding=0, layer_attr=None): +def img_pool_layer(input, + pool_size, + name=None, + num_channels=None, + pool_type=None, + stride=1, + padding=0, + layer_attr=None, + pool_size_y=None, + stride_y=None, + padding_y=None, + img_width=None): """ Image pooling Layer. @@ -1635,25 +1918,32 @@ def img_pool_layer(input, pool_size, name=None, .. _pooling: http://ufldl.stanford.edu/tutorial/supervised/Pooling/ - :param padding: pooling padding + :param padding: pooling padding width. :type padding: int + :param padding_y: pooling padding height. It's equal to padding by default. + :type padding_y: int|None :param name: name of pooling layer :type name: basestring. :param input: layer's input :type input: LayerOutput - :param pool_size: pooling size + :param pool_size: pooling window width :type pool_size: int + :param pool_size_y: pooling window height. It's eaqual to pool_size by default. + :type pool_size_y: int|None :param num_channels: number of input channel. :type num_channels: int - :param pool_type: pooling type. MaxPooling or AveragePooling. Default is + :param pool_type: pooling type. MaxPooling or AvgPooling. Default is MaxPooling. :type pool_type: BasePoolingType - :param stride: stride of pooling. + :param stride: stride width of pooling. :type stride: int - :param start: start position of pooling operation. - :type start: int + :param stride_y: stride height of pooling. It is equal to stride by default. + :type stride_y: int|None :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute + :param img_width: the width of input feature map. If it is None, the input feature + map should be square. + :type img_width: int|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -1666,47 +1956,141 @@ def img_pool_layer(input, pool_size, name=None, elif isinstance(pool_type, AvgPooling): pool_type.name = 'avg' - Layer( + type_name = pool_type.name + '-projection' \ + if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)) \ + else pool_type.name + + pool_size_y = pool_size if pool_size_y is None else pool_size_y + stride_y = stride if stride_y is None else stride_y + padding_y = padding if padding_y is None else padding_y + + l = Layer( name=name, type=LayerType.POOL_LAYER, - inputs=[Input(input.name, - pool=Pool( - pool_type=''.join([pool_type.name, '-projection']), - channels=num_channels, - size_x=pool_size, - start=start, - stride=stride, - padding=padding - ))], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POOL_LAYER, parents=[input], - num_filters=num_channels) - - -def __img_norm_layer__(name, input, size, norm_type, scale, power, - num_channels, blocked, layer_attr): + inputs=[ + Input( + input.name, + pool=Pool( + pool_type=type_name, + channels=num_channels, + size_x=pool_size, + start=None, + stride=stride, + padding=padding, + size_y=pool_size_y, + stride_y=stride_y, + padding_y=padding_y, + img_width=img_width)) + ], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.POOL_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + + +@wrap_name_default("spp") +@layer_support() +def spp_layer(input, + name=None, + num_channels=None, + pool_type=None, + pyramid_height=None, + img_width=None, + layer_attr=None): + """ + Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. + The details please refer to + `Kaiming He's paper `_. + + :param name: layer name. + :type name: basestring + :param input: layer's input. + :type input: LayerOutput + :param num_channels: number of input channel. + :type num_channels: int + :param pool_type: Pooling type. MaxPooling or AveragePooling. Default is MaxPooling. + :type scale: BasePoolingType + :param pyramid_height: pyramid height. + :type pyramid_height: int + :param img_width: the width of input feature map. If it is None, the input feature + map should be square. + :type img_width: int|None + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters - Layer( - name=name, type=LayerType.NORM_LAYER, inputs=Input( - input.name, norm=Norm(norm_type=norm_type, - channels=num_channels, size=size, - scale=scale, - pow=power, blocked=blocked) - ), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, layer_type=LayerType.NORM_LAYER, parents=[input], - num_filters=num_channels, img_norm_type=norm_type) + if pool_type is None: + pool_type = MaxPooling() + elif isinstance(pool_type, AvgPooling): + pool_type.name = 'avg' + + type_name = pool_type.name + if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)): + type_name += '-projection' + + l = Layer( + name=name, + type=LayerType.SPP_LAYER, + inputs=Input( + input.name, + spp=SpatialPyramidPool( + pool_type=type_name, + channels=num_channels, + pyramid_height=pyramid_height, + img_width=img_width)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.SPP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + + +def __img_norm_layer__(name, input, size, norm_type, scale, power, num_channels, + blocked, layer_attr): + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + + l = Layer( + name=name, + type=LayerType.NORM_LAYER, + inputs=Input( + input.name, + norm=Norm( + norm_type=norm_type, + channels=num_channels, + size=size, + scale=scale, + pow=power, + blocked=blocked)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.NORM_LAYER, + parents=[input], + num_filters=num_channels, + img_norm_type=norm_type, + size=l.config.size) @wrap_name_default("crmnorm") @layer_support() -def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, - name=None, num_channels=None, +def img_cmrnorm_layer(input, + size, + scale=0.0128, + power=0.75, + name=None, + num_channels=None, layer_attr=None): """ Response normalization across feature maps. @@ -1740,8 +2124,13 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, @wrap_act_default(act=ReluActivation()) @wrap_name_default("batch_norm") @layer_support(DROPOUT) -def batch_norm_layer(input, act=None, name=None, num_channels=None, - bias_attr=None, param_attr=None, layer_attr=None, +def batch_norm_layer(input, + act=None, + name=None, + num_channels=None, + bias_attr=None, + param_attr=None, + layer_attr=None, batch_norm_type=None, moving_average_fraction=0.9, use_global_stats=None): @@ -1825,23 +2214,25 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, num_channels = input.size assert (batch_norm_type is None) or (batch_norm_type == "batch_norm") or \ (batch_norm_type == "cudnn_batch_norm") - Layer( + l = Layer( name=name, - inputs=Input(input.name, - image=Image(channels=num_channels), - **param_attr.attr), + inputs=Input( + input.name, image=Image(channels=num_channels), **param_attr.attr), active_type=act.name, type=LayerType.BATCH_NORM_LAYER, batch_norm_type=batch_norm_type, bias=ParamAttr.to_bias(bias_attr), moving_average_fraction=moving_average_fraction, use_global_stats=use_global_stats, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.BATCH_NORM_LAYER, - parents=[input], activation=act, - num_filters=num_channels) + return LayerOutput( + name=name, + layer_type=LayerType.BATCH_NORM_LAYER, + parents=[input], + activation=act, + num_filters=num_channels, + size=l.config.size) @wrap_name_default() @@ -1876,18 +2267,16 @@ def sum_to_one_norm_layer(input, name=None, layer_attr=None): name=name, type=LayerType.SUM_TO_ONE_NORM_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], size=input.size) @wrap_name_default("addto") @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(DROPOUT) -def addto_layer(input, act=None, name=None, bias_attr=None, - layer_attr=None): +def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): """ AddtoLayer. @@ -1946,29 +2335,41 @@ def addto_layer(input, act=None, name=None, bias_attr=None, if each_input.num_filters is not None: num_filters = each_input.num_filters - Layer( - name=name, type=LayerType.ADDTO_LAYER, inputs=ipts_for_layer, + l = Layer( + name=name, + type=LayerType.ADDTO_LAYER, + inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.ADDTO_LAYER, parents=input, - activation=act, num_filters=num_filters) + return LayerOutput( + name, + LayerType.ADDTO_LAYER, + parents=input, + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_act_default(act=IdentityActivation()) @wrap_name_default("concat") @layer_support() -def concat_layer(input, act=None, name=None, layer_attr=None): +def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): """ Concat all input vector into one huge vector. Inputs can be list of LayerOutput or list of projection. + The example usage is: + + .. code-block:: python + + concat = concat_layer(input=[layer1, layer2]) + :param name: Layer name. :type name: basestring :param input: input layers or projections - :type input: list|tuple|collection.Sequence + :type input: list|tuple|collections.Sequence :param act: Activation type. :type act: BaseActivation :param layer_attr: Extra Layer Attribute. @@ -2007,18 +2408,22 @@ def __reduce_concat_type__(a, b): LayerOutput) return a - is_concat_layer = __is_type__(reduce(__reduce_concat_type__, - map(type, input)), LayerOutput) + is_concat_layer = __is_type__( + reduce(__reduce_concat_type__, map(type, input)), LayerOutput) - layer_type = (LayerType.CONCAT_LAYER if is_concat_layer - else LayerType.CONCAT_PROJ_LAYER) + layer_type = (LayerType.CONCAT_LAYER + if is_concat_layer else LayerType.CONCAT_PROJ_LAYER) + + if layer_type == LayerType.CONCAT_LAYER: + assert not bias_attr Layer( - name=name, type=layer_type, + name=name, + type=layer_type, inputs=[x.name for x in input] if is_concat_layer else input, active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + bias=ParamAttr.to_bias(bias_attr), + **ExtraLayerAttribute.to_kwargs(layer_attr)) sz = 0 for each_input in input: @@ -2028,14 +2433,20 @@ def __reduce_concat_type__(a, b): sz = None break - return LayerOutput(name, layer_type=layer_type, - parents=input if is_concat_layer else [ - x.origin for x in input], - activation=act, size=sz) - - -def memory(name, size, is_seq=False, boot_layer=None, - boot_bias=None, boot_bias_active_type=None, + return LayerOutput( + name, + layer_type=layer_type, + parents=input if is_concat_layer else [x.origin for x in input], + activation=act, + size=sz) + + +def memory(name, + size, + is_seq=False, + boot_layer=None, + boot_bias=None, + boot_bias_active_type=None, boot_with_const_id=None): """ The memory layers is a layer cross each time step. Reference this output @@ -2083,30 +2494,33 @@ def memory(name, size, is_seq=False, boot_layer=None, assert boot_layer is None or isinstance(boot_layer, LayerOutput) - agent_name = Memory(name, size, - is_seq, - boot_layer.name if boot_layer is not None else None, - boot_bias, - boot_bias_active_type.name, - boot_with_const_id) - - lout = LayerOutput(name=agent_name, size=size, - layer_type=LayerType.MEMORY, - parents=[boot_layer] if boot_layer is not None - else None) + agent_name = Memory(name, size, is_seq, boot_layer.name + if boot_layer is not None else None, boot_bias, + boot_bias_active_type.name, boot_with_const_id) + + lout = LayerOutput( + name=agent_name, + size=size, + layer_type=LayerType.MEMORY, + parents=[boot_layer] if boot_layer is not None else None) return lout @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act', - 'state_act'], - act=SigmoidActivation()) +@wrap_act_default( + param_names=['gate_act', 'state_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('lstm_step') @layer_support() -def lstm_step_layer(input, state, size, act=None, - name=None, gate_act=None, state_act=None, - bias_attr=None, layer_attr=None): +def lstm_step_layer(input, + state, + size, + act=None, + name=None, + gate_act=None, + state_act=None, + bias_attr=None, + layer_attr=None): """ LSTM Step Layer. It used in recurrent_group. The lstm equations are shown as follow. @@ -2173,24 +2587,32 @@ def lstm_step_layer(input, state, size, act=None, active_gate_type=gate_act.name, active_state_type=state_act.name, bias=ParamAttr.to_bias(bias_attr), - size=size, inputs=[input.name, state.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + size=size, + inputs=[input.name, state.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.LSTM_STEP_LAYER, - parents=[input, state], activation=act, - size=size, outputs=['default', 'state']) + return LayerOutput( + name=name, + layer_type=LayerType.LSTM_STEP_LAYER, + parents=[input, state], + activation=act, + size=size, + outputs=['default', 'state']) @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('gru_step') @layer_support() -def gru_step_layer(input, output_mem, size=None, act=None, - name=None, gate_act=None, - bias_attr=None, layer_attr=None): +def gru_step_layer(input, + output_mem, + size=None, + act=None, + name=None, + gate_act=None, + bias_attr=None, + layer_attr=None): """ :param input: @@ -2211,20 +2633,18 @@ def gru_step_layer(input, output_mem, size=None, act=None, Layer( name=name, type=LayerType.GRU_STEP_LAYER, - inputs=[ - input.name, - output_mem.name - ], + inputs=[input.name, output_mem.name], bias=ParamAttr.to_bias(bias_attr), size=size, active_type=act.name, active_gate_type=gate_act.name, - **ExtraAttr.to_kwargs(layer_attr) - ) + **ExtraAttr.to_kwargs(layer_attr)) return LayerOutput( - name=name, layer_type=LayerType.GRU_STEP_LAYER, + name=name, + layer_type=LayerType.GRU_STEP_LAYER, parents=[input, output_mem], - size=size, activation=act) + size=size, + activation=act) @wrap_name_default() @@ -2252,13 +2672,19 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): ' The get output name is %s, which not' \ ' in %s' % ( arg_name, ",".join(input.outputs)) - Layer(name=name, type=LayerType.GET_OUTPUT_LAYER, - inputs=[Input(input.name, input_layer_argument=arg_name)], - size=input.size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.GET_OUTPUT_LAYER, + inputs=[Input( + input.name, input_layer_argument=arg_name)], + size=input.size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.GET_OUTPUT_LAYER, - parents=[input], size=input.size) + return LayerOutput( + name=name, + layer_type=LayerType.GET_OUTPUT_LAYER, + parents=[input], + size=input.size) @wrap_name_default() @@ -2266,8 +2692,13 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): @wrap_bias_attr_default() @wrap_param_attr_default() @layer_support() -def recurrent_layer(input, act=None, bias_attr=None, - param_attr=None, name=None, reverse=False, layer_attr=None): +def recurrent_layer(input, + act=None, + bias_attr=None, + param_attr=None, + name=None, + reverse=False, + layer_attr=None): """ Simple recurrent unit layer. It is just a fully connect layer through both time and neural network. @@ -2302,16 +2733,21 @@ def recurrent_layer(input, act=None, bias_attr=None, :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, - type=LayerType.RECURRENT_LAYER, - inputs=Input(input.name, **param_attr.attr), - active_type=act.name, - bias=ParamAttr.to_bias(bias_attr), - reversed=reverse, - **ExtraAttr.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.RECURRENT_LAYER, - parents=[input], size=input.size, activation=act, - reverse=reverse) + Layer( + name=name, + type=LayerType.RECURRENT_LAYER, + inputs=Input(input.name, **param_attr.attr), + active_type=act.name, + bias=ParamAttr.to_bias(bias_attr), + reversed=reverse, + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.RECURRENT_LAYER, + parents=[input], + size=input.size, + activation=act, + reverse=reverse) class StaticInput(object): @@ -2347,7 +2783,7 @@ def __init__(self, input): @wrap_name_default("recurrent_group") -def recurrent_group(step, input, reverse=False, name=None): +def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): """ Recurrent layer group is an extremely flexible recurrent unit in PaddlePaddle. As long as the user defines the calculation done within a @@ -2401,6 +2837,17 @@ def step(input): :param reverse: If reverse is set true, the recurrent unit will process the input sequence in a reverse order. :type reverse: bool + + :param targetInlink: the input layer which share info with layer group's output + + Param input specifies multiple input layers. For + SubsequenceInput inputs, config should assign one input + layer that share info(the number of sentences and the number + of words in each sentence) with all layer group's outputs. + targetInlink should be one of the layer group's input. + + :type targetInlink: LayerOutput|SubsequenceInput + :return: LayerOutput object. :rtype: LayerOutput """ @@ -2419,6 +2866,20 @@ def is_in_links(x): in_links = filter(is_in_links, input) + def targetInlink_in_inlinks(): + for inlink in in_links: + if isinstance(inlink, SubsequenceInput): + if targetInlink == inlink.input: + return True + elif targetInlink == inlink: + return True + return False + + assert (targetInlink == None or targetInlink_in_inlinks()) + targetInlinkName = None if targetInlink == None \ + else targetInlink.name if isinstance(targetInlink, LayerOutput) \ + else targetInlink.input.name + contains_sub_seq = [False] def map_in_links(x): @@ -2429,8 +2890,10 @@ def map_in_links(x): return x.name RecurrentLayerGroupWithoutOutLinksBegin( - name=name, in_links=map(map_in_links, in_links), - seq_reversed=reverse) + name=name, + in_links=map(map_in_links, in_links), + seq_reversed=reverse, + target_inlinkname=targetInlinkName) in_args = [] for each_input in input: assert is_single_input(each_input) @@ -2440,12 +2903,15 @@ def map_in_links(x): in_args.append(each_input.input) else: mem_name = "__%s_memory__" % each_input.input.name - mem = memory(name=mem_name, - is_seq=each_input.is_seq, - size=each_input.input.size, - boot_layer=each_input.input) - with mixed_layer(name=mem_name, size=each_input.input.size, - act=IdentityActivation()) as mix: + mem = memory( + name=mem_name, + is_seq=each_input.is_seq, + size=each_input.input.size, + boot_layer=each_input.input) + with mixed_layer( + name=mem_name, + size=each_input.input.size, + act=IdentityActivation()) as mix: mix += identity_projection(mem) in_args.append(mem) @@ -2487,14 +2953,15 @@ def after_real_step(self, input): return maxid_layer(input=input, name='__beam_search_predict__') def before_real_step(self): - predict_id = memory(name='__beam_search_predict__', - size=self.size, - boot_with_const_id=self.bos_id) - - trg_emb = embedding_layer(input=predict_id, - size=self.embedding_size, - param_attr=ParamAttr( - name=self.embedding_name)) + predict_id = memory( + name='__beam_search_predict__', + size=self.size, + boot_with_const_id=self.bos_id) + + trg_emb = embedding_layer( + input=predict_id, + size=self.embedding_size, + param_attr=ParamAttr(name=self.embedding_name)) return trg_emb def __init__(self, size, embedding_name, embedding_size): @@ -2527,13 +2994,16 @@ def maxid_layer(input, name=None, layer_attr=None): """ assert isinstance(input, LayerOutput) - Layer(name=name, - type='maxid', - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.MAXID_LAYER, - parents=[input]) + l = Layer( + name=name, + type='maxid', + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.MAXID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() @@ -2562,13 +3032,16 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): assert isinstance(input1, LayerOutput) assert isinstance(input2, LayerOutput) - Layer(name=name, - type="out_prod", - inputs=[input1.name, input2.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.OUT_PROD_LAYER, - parents=[input1, input2]) + l = Layer( + name=name, + type=LayerType.OUT_PROD_LAYER, + inputs=[input1.name, input2.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.OUT_PROD_LAYER, + parents=[input1, input2], + size=l.config.size) @wrap_name_default() @@ -2597,18 +3070,27 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, - type=LayerType.EOSID_LAYER, - eos_id=eos_id, - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.EOSID_LAYER, - parents=[input]) + l = Layer( + name=name, + type=LayerType.EOSID_LAYER, + eos_id=eos_id, + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.EOSID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() -def beam_search(step, input, bos_id, eos_id, beam_size, - max_length=500, name=None, +def beam_search(step, + input, + bos_id, + eos_id, + beam_size, + max_length=500, + name=None, num_results_per_sample=None): """ Beam search is a heuristic search algorithm used in sequence generation. @@ -2682,8 +3164,7 @@ def rnn_step(input): if num_results_per_sample > beam_size: logger.warning("num_results_per_sample should be less than beam_size") - if isinstance(input, StaticInput) or isinstance(input, - BaseGeneratedInput): + if isinstance(input, StaticInput) or isinstance(input, BaseGeneratedInput): input = [input] generated_input_index = -1 @@ -2708,11 +3189,12 @@ def rnn_step(input): def __real_step__(*args): eos_name = "__%s_eos_layer__" % name - RecurrentLayerGroupSetGenerator(Generator( - eos_layer_name=eos_name, - max_num_frames=max_length, - beam_size=beam_size, - num_results_per_sample=num_results_per_sample)) + RecurrentLayerGroupSetGenerator( + Generator( + eos_layer_name=eos_name, + max_num_frames=max_length, + beam_size=beam_size, + num_results_per_sample=num_results_per_sample)) args = list(args) args.insert(generated_input_index, gipt.before_real_step()) @@ -2723,35 +3205,63 @@ def __real_step__(*args): return predict - tmp = recurrent_group(step=__real_step__, input=real_input, reverse=False, - name=name) - + tmp = recurrent_group( + step=__real_step__, input=real_input, reverse=False, name=name) + return tmp +def __cost_input__(input, label, weight=None): + """ + inputs and parents for cost layers. + """ + ipts = [Input(input.name), Input(label.name)] + parents = [input, label] + if weight is not None: + assert weight.layer_type == LayerType.DATA + ipts.append(Input(weight.name)) + parents.append(weight) + return ipts, parents + + @wrap_name_default() -def regression_cost(input, label, cost='square_error', name=None): +@layer_support() +def regression_cost(input, label, weight=None, name=None, layer_attr=None): """ Regression Layer. TODO(yuyang18): Complete this method. :param name: layer name. + :type name: basestring :param input: Network prediction. + :type input: LayerOutput :param label: Data label. - :param cost: Cost method. + :type label: LayerOutput + :param weight: The weight affects the cost, namely the scale of cost. + It is an optional argument. + :type weight: LayerOutput + :param layer_attr: layer's extra attribute. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. + :rtype: LayerOutput """ - Layer(inputs=[Input(input.name), Input(label.name)], type=cost, name=name) - return LayerOutput( - name, LayerType.COST, parents=[input, label] - ) + ipts, parents = __cost_input__(input, label, weight) + + Layer( + inputs=ipts, + type="square_error", + name=name, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput(name, LayerType.COST, parents=parents, size=1) @wrap_name_default("cost") @layer_support() -def classification_cost(input, label, name=None, - cost="multi-class-cross-entropy", +def classification_cost(input, + label, + weight=None, + name=None, evaluator=classification_error_evaluator, layer_attr=None): """ @@ -2763,8 +3273,9 @@ def classification_cost(input, label, name=None, :type input: LayerOutput :param label: label layer name. data_layer often. :type label: LayerOutput - :param cost: cost method. - :type cost: basestring + :param weight: The weight affects the cost, namely the scale of cost. + It is an optional argument. + :type weight: LayerOutput :param evaluator: Evaluator method. :param layer_attr: layer's extra attribute. :type layer_attr: ExtraLayerAttribute @@ -2774,8 +3285,14 @@ def classification_cost(input, label, name=None, assert input.layer_type != LayerType.DATA assert isinstance(input.activation, SoftmaxActivation) assert label.layer_type == LayerType.DATA - Layer(name=name, type=cost, inputs=[Input(input.name), Input(label.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr)) + + ipts, parents = __cost_input__(input, label, weight) + + Layer( + name=name, + type="multi-class-cross-entropy", + inputs=ipts, + **ExtraLayerAttribute.to_kwargs(layer_attr)) def __add_evaluator__(e): assert callable(e) @@ -2786,7 +3303,7 @@ def __add_evaluator__(e): assert isinstance(e.for_classification, bool) assert e.for_classification - e(name=e.__name__, input=input, label=label) + e(name=e.__name__, input=input, label=label, weight=weight) if not isinstance(evaluator, collections.Sequence): evaluator = [evaluator] @@ -2794,12 +3311,19 @@ def __add_evaluator__(e): for each_evaluator in evaluator: __add_evaluator__(each_evaluator) - return LayerOutput(name, LayerType.COST, parents=[input, label]) + return LayerOutput(name, LayerType.COST, parents=parents, size=1) -def conv_operator(img, filter, filter_size, num_filters, - num_channel=None, stride=1, padding=0, - filter_size_y=None, stride_y=None, padding_y=None): +def conv_operator(img, + filter, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used in mixed_layer. And conv_op takes two inputs to perform convolution. @@ -2828,8 +3352,8 @@ def conv_operator(img, filter, filter_size, num_filters, :type filter_size_y: int :param num_filters: channel of output data. :type num_filters: int - :param num_channel: channel of input data. - :type num_channel: int + :param num_channels: channel of input data. + :type num_channels: int :param stride: The x dimension of the stride. :type stride: int :param stride_y: The y dimension of the stride. @@ -2848,29 +3372,139 @@ def conv_operator(img, filter, filter_size, num_filters, if padding_y is None: padding_y = padding - if num_channel is None: - num_channel = img.num_filters + if num_channels is None: + num_channels = img.num_filters assert isinstance(filter, LayerOutput) if filter.size is not None: - filter.size = filter_size * filter_size_y * num_filters * num_channel - - op = ConvOperator(input_layer_names=[img.name, filter.name], - num_filters=num_filters, - conv_conf=Conv(filter_size=filter_size, - padding=padding, - stride=stride, - channels=num_channel, - filter_size_y=filter_size_y, - padding_y=padding_y, - stride_y=stride_y, - groups=1)) + filter.size = filter_size * filter_size_y * num_filters * num_channels + + op = ConvOperator( + input_layer_names=[img.name, filter.name], + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=1)) op.origin = [img, filter] return op +@wrap_param_attr_default() +def conv_projection(input, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None, + groups=1, + param_attr=None): + """ + ConvProjection with a layer as input. + It performs element-wise multiplication with weight. + + Different from img_conv_layer and conv_op, conv_projection is an Projection, + which can be used in mixed_layer and conat_layer. It use cudnn to implement + conv and only support GPU mode. + + The example usage is: + + .. code-block:: python + + proj = conv_projection(img=input1, + filter_size=3, + num_filters=64, + num_channels=64) + + :param input: input layer + :type input: LayerOutput + :param filter_size: The x dimension of a filter kernel. + :type filter_size: int + :param filter_size_y: The y dimension of a filter kernel. Since + PaddlePaddle now supports rectangular filters, + the filter's shape can be (filter_size, filter_size_y). + :type filter_size_y: int + :param num_filters: channel of output data. + :type num_filters: int + :param num_channels: channel of input data. + :type num_channels: int + :param stride: The x dimension of the stride. + :type stride: int + :param stride_y: The y dimension of the stride. + :type stride_y: int + :param padding: The x dimension of padding. + :type padding: int + :param padding_y: The y dimension of padding. + :type padding_y: int + :param groups: The group number. + :type groups: int + :param param_attr: Convolution param attribute. None means default attribute + :type param_attr: ParameterAttribute + :return: A DotMulProjection Object. + :rtype: DotMulProjection + """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + + if filter_size_y is None: + if isinstance(filter_size, collections.Sequence): + assert len(filter_size) == 2 + filter_size, filter_size_y = filter_size + else: + filter_size_y = filter_size + + if stride_y is None: + if isinstance(stride, collections.Sequence): + assert len(stride) == 2 + stride, stride_y = stride + else: + stride_y = stride + + if padding_y is None: + if isinstance(padding, collections.Sequence): + assert len(padding) == 2 + padding, padding_y = padding + else: + padding_y = padding + + if param_attr.attr.get('initial_smart'): + # special initial for conv layers. + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 + param_attr.attr["initial_mean"] = 0.0 + param_attr.attr["initial_std"] = init_w + param_attr.attr["initial_strategy"] = 0 + param_attr.attr["initial_smart"] = False + + proj = ConvProjection( + input_layer_name=input.name, + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=groups), + **param_attr.attr) + + proj.origin = input + return proj + + @wrap_name_default() -def conv_shift_layer(a, b, name=None): +@layer_support() +def conv_shift_layer(a, b, name=None, layer_attr=None): """ This layer performs cyclic convolution for two input. For example: - a[in]: contains M elements. @@ -2899,6 +3533,8 @@ def conv_shift_layer(a, b, name=None): :type a: LayerOutput :param b: input layer b :type b: LayerOutput + :param layer_attr: layer's extra attribute. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput """ @@ -2908,10 +3544,10 @@ def conv_shift_layer(a, b, name=None): name=name, type=LayerType.CONV_SHIFT_LAYER, inputs=[a.name, b.name], - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], - size=a.size) + return LayerOutput( + name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], size=a.size) @wrap_name_default() @@ -2919,8 +3555,14 @@ def conv_shift_layer(a, b, name=None): @wrap_bias_attr_default() @wrap_act_default(act=LinearActivation()) @layer_support(ERROR_CLIPPING, DROPOUT) -def tensor_layer(a, b, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def tensor_layer(a, + b, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ This layer performs tensor operation for two input. For example, each sample: @@ -2969,23 +3611,28 @@ def tensor_layer(a, b, size, act=None, name=None, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(a.name, **param_attr.attr), - Input(b.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[a, b], - activation=act, size=size) + inputs=[Input(a.name, **param_attr.attr), Input(b.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TENSOR_LAYER, parents=[a, b], activation=act, size=size) @wrap_name_default() @wrap_param_attr_default() @wrap_bias_attr_default() @wrap_act_default() -def selective_fc_layer(input, select, size, act=None, name=None, +@layer_support() +def selective_fc_layer(input, + select, + size, + act=None, + name=None, pass_generation=False, has_selected_colums=True, mul_ratio=0.02, - param_attr=None, bias_attr=None, layer_attr=None): + param_attr=None, + bias_attr=None, + layer_attr=None): """ Selectived fully connected layer. Different from fc_layer, the output of this layer maybe sparse. It requires an additional input to indicate @@ -3035,8 +3682,9 @@ def selective_fc_layer(input, select, size, act=None, name=None, if select.size is not None: assert select.size == size Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)] + [select.name], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ] + [select.name], name=name, type=LayerType.SEL_FC_LAYER, size=size, @@ -3045,15 +3693,18 @@ def selective_fc_layer(input, select, size, act=None, name=None, selective_fc_pass_generation=pass_generation, has_selected_colums=has_selected_colums, selective_fc_full_mul_ratio=mul_ratio, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEL_FC_LAYER, list(input) + [select], - activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEL_FC_LAYER, + list(input) + [select], + activation=act, + size=size) @wrap_name_default() -def sampling_id_layer(input, name=None): +@layer_support() +def sampling_id_layer(input, name=None, layer_attr=None): """ A layer for sampling id from multinomial distribution from the input layer. Sampling one id for one sample. @@ -3068,19 +3719,27 @@ def sampling_id_layer(input, name=None): :type input: LayerOutput :param name: The Layer Name. :type name: basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ - Layer( + l = Layer( name=name, type=LayerType.SAMPLING_ID_LAYER, inputs=[Input(input.name)], - ) - return LayerOutput(name, LayerType.SAMPLING_ID_LAYER, input) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SAMPLING_ID_LAYER, input, size=l.config.size) @wrap_name_default() -def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): +@layer_support() +def slope_intercept_layer(input, + name=None, + slope=1.0, + intercept=0.0, + layer_attr=None): """ This layer for applying a slope and an intercept to the input element-wise. There is no activation and weight. @@ -3102,6 +3761,8 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): :type slope: float. :param intercept: the offset. :type intercept: float. + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3111,12 +3772,14 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): slope=slope, intercept=intercept, inputs=[Input(input.name)], - ) - return LayerOutput(name, LayerType.SLOPE_INTERCEPT_LAYER, input) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SLOPE_INTERCEPT_LAYER, input, size=input.size) @wrap_name_default() -def linear_comb_layer(weights, vectors, size=None, name=None): +@layer_support() +def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None): """ A layer for weighted sum of vectors takes two inputs. - Input: size of weights is M @@ -3157,6 +3820,8 @@ def linear_comb_layer(weights, vectors, size=None, name=None): :type size: int :param name: The Layer Name. :type name: basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3164,7 +3829,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None): if vectors.size is not None and weights.size is not None: assert vectors.size % weights.size == 0 if size is None: - size = vectors.size / weights.size + size = vectors.size / weights.size else: assert size == vectors.size / weights.size Layer( @@ -3172,27 +3837,29 @@ def linear_comb_layer(weights, vectors, size=None, name=None): type=LayerType.LINEAR_COMBINATION_LAYER, size=size, inputs=[Input(weights.name), Input(vectors.name)], - ) - return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, - [weights, vectors], size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size) convex_comb_layer = linear_comb_layer @wrap_name_default() +@layer_support() def block_expand_layer(input, - channel=0, block_x=0, block_y=0, stride_x=0, stride_y=0, padding_x=0, padding_y=0, - name=None): + num_channels=None, + name=None, + layer_attr=None): """ Expand feature map to minibatch matrix. - - matrix width is: block_y * block_x * channel + - matrix width is: block_y * block_x * num_channels - matirx height is: outputH * outputW .. math:: @@ -3204,7 +3871,7 @@ def block_expand_layer(input, The expand method is the same with ExpandConvLayer, but saved the transposed value. After expanding, output.sequenceStartPositions will store timeline. The number of time steps are outputH * outputW and the dimension of each - time step is block_y * block_x * channel. This layer can be used after + time step is block_y * block_x * num_channels. This layer can be used after convolution neural network, and before recurrent neural network. The simple usage is: @@ -3212,7 +3879,7 @@ def block_expand_layer(input, .. code-block:: python block_expand = block_expand_layer(input, - channel=128, + num_channels=128, stride_x=1, stride_y=1, block_x=1, @@ -3220,8 +3887,8 @@ def block_expand_layer(input, :param input: The input layer. :type input: LayerOutput - :param channel: The channel number of input layer. - :type channel: int + :param num_channels: The channel number of input layer. + :type num_channels: int|None :param block_x: The width of sub block. :type block_x: int :param block_y: The width of sub block. @@ -3236,27 +3903,110 @@ def block_expand_layer(input, :type padding_y: int :param name: The name of this layer, which can not specify. :type name: None|basestring. + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, - input=Input(input.name, - block_expand=BlockExpand(channels=channel, - block_x=block_x, - block_y=block_y, - stride_x=stride_x, - stride_y=stride_y, - padding_x=padding_x, - padding_y=padding_y) - ), - type=LayerType.BLOCK_EXPAND, - ) + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + l = Layer( + name=name, + inputs=Input( + input.name, + block_expand=BlockExpand( + channels=num_channels, + block_x=block_x, + block_y=block_y, + stride_x=stride_x, + stride_y=stride_y, + padding_x=padding_x, + padding_y=padding_y)), + type=LayerType.BLOCK_EXPAND, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + + return LayerOutput( + name, LayerType.BLOCK_EXPAND, parents=[input], size=l.config.size) + + +@wrap_name_default() +@layer_support() +def maxout_layer(input, + groups, + num_channels=None, + size_x=None, + size_y=None, + name=None, + layer_attr=None): + """ + A layer to do max out on conv layer output. + - Input: output of a conv layer. + - Output: feature map size same as input. Channel is (input channel) / groups. + + So groups should be larger than 1, and the num of channels should be able + to devided by groups. - return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input]) + Please refer to Paper: + - Maxout Networks: http://www.jmlr.org/proceedings/papers/v28/goodfellow13.pdf + - Multi-digit Number Recognition from Street View \ + Imagery using Deep Convolutional Neural Networks: \ + https://arxiv.org/pdf/1312.6082v4.pdf + + The simple usage is: + + .. code-block:: python + + maxout = maxout_layer(input, + num_channels=128, + groups=4) + + :param input: The input layer. + :type input: LayerOutput + :param num_channels: The channel number of input layer. If None will be set + automatically from previous output. + :type num_channels: int|None + :param groups: The group number of input layer. + :type groups: int + :param size_x: conv output width. If None will be set + automatically from previous output. + :type size_x: int|None + :param size_y: conv output height. If None will be set + automatically from previous output. + :type size_y: int|None + :param name: The name of this layer, which can not specify. + :type name: None|basestring. + :param layer_attr: Extra Layer attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + assert input.layer_type == LayerType.CONV_LAYER + assert isinstance(input.activation, LinearActivation) + assert groups > 1 + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + assert num_channels % groups == 0 + l = Layer( + name=name, + inputs=Input( + input.name, maxout=MaxOut( + channels=num_channels, groups=groups)), + type=LayerType.MAXOUT, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.MAXOUT, parents=[input], size=l.config.size) @wrap_name_default() -def ctc_layer(input, label, size=None, name=None, norm_by_times=False): +@layer_support() +def ctc_layer(input, + label, + size=None, + name=None, + norm_by_times=False, + layer_attr=None): """ Connectionist Temporal Classification (CTC) is designed for temporal classication task. That is, for sequence labeling problems where the @@ -3293,6 +4043,8 @@ def ctc_layer(input, label, size=None, name=None, norm_by_times=False): :type name: basestring|None :param norm_by_times: Whether to normalization by times. False by default. :type norm_by_times: bool + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3308,14 +4060,21 @@ def ctc_layer(input, label, size=None, name=None, norm_by_times=False): type=LayerType.CTC_LAYER, size=size, norm_by_times=norm_by_times, - inputs=[input.name, label.name] - ) + inputs=[input.name, label.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) @wrap_name_default() @wrap_param_attr_default() -def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): +@layer_support() +def crf_layer(input, + label, + size=None, + weight=None, + param_attr=None, + name=None, + layer_attr=None): """ A layer for calculating the cost of sequential conditional random field model. @@ -3341,6 +4100,8 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): :type param_attr: ParameterAttribute :param name: The name of this layers. It is not necessary. :type name: None|basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3354,8 +4115,7 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): else: assert size == input.size - ipts = [Input(input.name, **param_attr.attr), - Input(label.name)] + ipts = [Input(input.name, **param_attr.attr), Input(label.name)] if weight is not None: ipts.append(Input(weight.name)) @@ -3364,16 +4124,25 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): type=LayerType.CRF_LAYER, size=size, inputs=ipts, - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input, label] if weight is not None: parents.append(weight) - return LayerOutput(name, LayerType.CRF_LAYER, parents, size=size) + # The size for LayerOutput means the dimension of the output. + # It's different from the meaning of crf layer, which is the number of + # classes. + return LayerOutput(name, LayerType.CRF_LAYER, parents, size=1) @wrap_name_default() @wrap_param_attr_default() -def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): +@layer_support() +def crf_decoding_layer(input, + size, + label=None, + param_attr=None, + name=None, + layer_attr=None): """ A layer for calculating the decoding sequence of sequential conditional random field model. The decoding sequence is stored in output.ids. @@ -3391,6 +4160,8 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): :type param_attr: ParameterAttribute :param name: The name of this layers. It is not necessary. :type name: None|basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3407,11 +4178,99 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): type=LayerType.CRF_DECODING_LAYER, size=size, inputs=ipts, - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input] if label is not None: parents.append(label) - return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=size) + # The size for LayerOutput means the dimension of the output. + # It's different from the meaning of crf layer, which is the number of + # classes. + return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1) + + +@wrap_bias_attr_default(has_bias=True) +@wrap_name_default() +@layer_support() +def nce_layer(input, + label, + num_classes, + weight=None, + num_neg_samples=10, + neg_distribution=None, + name=None, + bias_attr=None, + layer_attr=None): + """ + Noise-contrastive estimation. + Implements the method in the following paper: + A fast and simple algorithm for training neural probabilistic language models. + + The example usage is: + + .. code-block:: python + + cost = nce_layer(input=layer1, label=layer2, weight=layer3, + num_classes=3, neg_distribution=[0.1,0.3,0.6]) + + :param name: layer name + :type name: basestring + :param input: input layers. It could be a LayerOutput of list/tuple of LayerOutput. + :type input: LayerOutput|list|tuple|collections.Sequence + :param label: label layer + :type label: LayerOutput + :param weight: weight layer, can be None(default) + :type weight: LayerOutput + :param num_classes: number of classes. + :type num_classes: int + :param num_neg_samples: number of negative samples. Default is 10. + :type num_neg_samples: int + :param neg_distribution: The distribution for generating the random negative labels. + A uniform distribution will be used if not provided. + If not None, its length must be equal to num_classes. + :type neg_distribution: list|tuple|collections.Sequence|None + :param bias_attr: Bias parameter attribute. True if no bias. + :type bias_attr: ParameterAttribute|None|False + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :return: layer name. + :rtype: LayerOutput + """ + if isinstance(input, LayerOutput): + input = [input] + assert isinstance(input, collections.Sequence) + assert isinstance(label, LayerOutput) + assert label.layer_type == LayerType.DATA + if neg_distribution is not None: + assert isinstance(neg_distribution, collections.Sequence) + assert len(neg_distribution) == num_classes + assert sum(neg_distribution) == 1 + + ipts_for_layer = [] + parents = [] + for each_input in input: + assert isinstance(each_input, LayerOutput) + ipts_for_layer.append(each_input.name) + parents.append(each_input) + ipts_for_layer.append(label.name) + parents.append(label) + + if weight is not None: + assert isinstance(weight, LayerOutput) + assert weight.layer_type == LayerType.DATA + ipts_for_layer.append(weight.name) + parents.append(weight) + + l = Layer( + name=name, + type=LayerType.NCE_LAYER, + num_classes=num_classes, + neg_sampling_dist=neg_distribution, + num_neg_samples=num_neg_samples, + inputs=ipts_for_layer, + bias=ParamAttr.to_bias(bias_attr), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.NCE_LAYER, parents=parents, size=l.config.size) """ @@ -3420,7 +4279,14 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): @wrap_name_default() -def rank_cost(left, right, label, weight=None, name=None, coeff=1.0): +@layer_support() +def rank_cost(left, + right, + label, + weight=None, + name=None, + coeff=1.0, + layer_attr=None): """ A cost Layer for learning to rank using gradient descent. Details can refer to `papers