diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..1ff0c423042 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,63 @@ +############################################################################### +# Set default behavior to automatically normalize line endings. +############################################################################### +* text=auto + +############################################################################### +# Set default behavior for command prompt diff. +# +# This is need for earlier builds of msysgit that does not have it on by +# default for csharp files. +# Note: This is only used by command line +############################################################################### +#*.cs diff=csharp + +############################################################################### +# Set the merge driver for project and solution files +# +# Merging from the command prompt will add diff markers to the files if there +# are conflicts (Merging from VS is not affected by the settings below, in VS +# the diff markers are never inserted). Diff markers may cause the following +# file extensions to fail to load in VS. An alternative would be to treat +# these files as binary and thus will always conflict and require user +# intervention with every merge. To do so, just uncomment the entries below +############################################################################### +#*.sln merge=binary +#*.csproj merge=binary +#*.vbproj merge=binary +#*.vcxproj merge=binary +#*.vcproj merge=binary +#*.dbproj merge=binary +#*.fsproj merge=binary +#*.lsproj merge=binary +#*.wixproj merge=binary +#*.modelproj merge=binary +#*.sqlproj merge=binary +#*.wwaproj merge=binary + +############################################################################### +# behavior for image files +# +# image files are treated as binary by default. +############################################################################### +#*.jpg binary +#*.png binary +#*.gif binary + +############################################################################### +# diff behavior for common document formats +# +# Convert binary document formats to text before diffing them. This feature +# is only available from the command line. Turn it on by uncommenting the +# entries below. +############################################################################### +#*.doc diff=astextplain +#*.DOC diff=astextplain +#*.docx diff=astextplain +#*.DOCX diff=astextplain +#*.dot diff=astextplain +#*.DOT diff=astextplain +#*.pdf diff=astextplain +#*.PDF diff=astextplain +#*.rtf diff=astextplain +#*.RTF diff=astextplain diff --git a/.gitignore b/.gitignore index eff292b7f61..2ead5c521fc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,15 +5,21 @@ *.lo *.o *.cuo +*.obj # Compiled Dynamic libraries *.so *.dylib +*.dll # Compiled Static libraries *.lai *.la *.a +*.lib + +# Compiled Executables +*.exe # Compiled protocol buffers *.pb.h @@ -22,6 +28,7 @@ # Compiled python *.pyc +*.pyd # Compiled MATLAB *.mex* @@ -97,3 +104,11 @@ LOCK LOG* CURRENT MANIFEST-* + +#Visual Studio files +*.user +*.suo +*.sdf +*.opensdf +*.pdb +*.props \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 08f56a33a59..db316eeab57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,12 @@ cmake_minimum_required(VERSION 2.8.7) +if(MSVC) + # CMake 3.4 introduced a WINDOWS_EXPORT_ALL_SYMBOLS target property that makes it possible to + # build shared libraries without using the usual declspec() decoration. + # See: https://blog.kitware.com/create-dlls-on-windows-without-declspec-using-new-cmake-export-all-feature/ + # and https://cmake.org/cmake/help/v3.5/prop_tgt/WINDOWS_EXPORT_ALL_SYMBOLS.html + # for details. + cmake_minimum_required(VERSION 3.4) +endif() if(POLICY CMP0046) cmake_policy(SET CMP0046 NEW) endif() @@ -25,15 +33,22 @@ include(cmake/Targets.cmake) include(cmake/Misc.cmake) include(cmake/Summary.cmake) include(cmake/ConfigGen.cmake) +include(cmake/WindowsCreateLinkHeader.cmake) +include(cmake/TargetResolvePrerequesites.cmake) # ---[ Options caffe_option(CPU_ONLY "Build Caffe without CUDA support" OFF) # TODO: rename to USE_CUDA caffe_option(USE_CUDNN "Build Caffe with cuDNN library support" ON IF NOT CPU_ONLY) caffe_option(USE_NCCL "Build Caffe with NCCL library support" OFF) -caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON) +if(MSVC) + # default to static libs + caffe_option(BUILD_SHARED_LIBS "Build shared libraries" OFF) +else() + caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON) +endif() caffe_option(BUILD_python "Build Python wrapper" ON) set(python_version "2" CACHE STRING "Specify which Python version to use") -caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE) +caffe_option(BUILD_matlab "Build Matlab wrapper" OFF) caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE) caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON) caffe_option(USE_OPENCV "Build with OpenCV support" ON) @@ -41,6 +56,25 @@ caffe_option(USE_LEVELDB "Build with levelDB" ON) caffe_option(USE_LMDB "Build with lmdb" ON) caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF) caffe_option(USE_OPENMP "Link with OpenMP (when your BLAS wants OpenMP and you get linker errors)" OFF) +caffe_option(protobuf_MODULE_COMPATIBLE "Make the protobuf-config.cmake compatible with the module mode" ON IF MSVC) +caffe_option(COPY_PREREQUISITES "Copy the prerequisites next to each executable or shared library directory" ON IF MSVC) +caffe_option(INSTALL_PREREQUISITES "Install the prerequisites next to each executable or shared library directory" ON IF MSVC) + +if(MSVC AND BUILD_SHARED_LIBS) + if(CMAKE_GENERATOR MATCHES "Visual Studio") + # see issue https://gitlab.kitware.com/cmake/cmake/issues/16552#note_215236 + message(FATAL_ERROR "The Visual Studio generator cannot build a shared library. Use the Ninja generator instead.") + endif() + # Some tests (solver tests) fail when caffe is built as a shared library. The problem comes + # from protobuf that has a global static empty_string_ variable. Since caffe and test.testbin + # link to a static protobuf library both end up with their own instance of the empty_string_ + # variable. This causes some SEH exception to occur. In practice if the caffe executable does not link + # to protobuf this problem should not happen. Use at your own risk. + message(WARNING "Some tests (solvers) will fail when building as a shared library with MSVC") +endif() + +# ---[ Prebuild dependencies on windows +include(cmake/WindowsDownloadPrebuiltDependencies.cmake) # ---[ Dependencies include(cmake/Dependencies.cmake) @@ -96,11 +130,16 @@ add_subdirectory(matlab) add_subdirectory(docs) # ---[ Linter target -add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake) +add_custom_target(lint COMMAND ${CMAKE_COMMAND} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake) # ---[ pytest target if(BUILD_python) - add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python ) + if(UNIX) + set(python_executable python${python_version}) + else() + set(python_executable ${PYTHON_EXECUTABLE}) + endif() + add_custom_target(pytest COMMAND ${python_executable} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python ) add_dependencies(pytest pycaffe) endif() diff --git a/README.md b/README.md index 0ae3616b4a6..e93d03b2c4b 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,140 @@ -# Caffe +# Windows Caffe -[![Build Status](https://travis-ci.org/BVLC/caffe.svg?branch=master)](https://travis-ci.org/BVLC/caffe) -[![License](https://img.shields.io/badge/license-BSD-blue.svg)](LICENSE) +**This is an experimental, community based branch led by Guillaume Dumont (@willyd). It is a work-in-progress.** -Caffe is a deep learning framework made with expression, speed, and modularity in mind. -It is developed by Berkeley AI Research ([BAIR](http://bair.berkeley.edu))/The Berkeley Vision and Learning Center (BVLC) and community contributors. +This branch of Caffe ports the framework to Windows. -Check out the [project site](http://caffe.berkeleyvision.org) for all the details like +[![Travis Build Status](https://api.travis-ci.org/BVLC/caffe.svg?branch=windows)](https://travis-ci.org/BVLC/caffe) Travis (Linux build) -- [DIY Deep Learning for Vision with Caffe](https://docs.google.com/presentation/d/1UeKXVgRvvxg9OUdh_UiC5G71UMscNPlvArsWER41PsU/edit#slide=id.p) -- [Tutorial Documentation](http://caffe.berkeleyvision.org/tutorial/) -- [BAIR reference models](http://caffe.berkeleyvision.org/model_zoo.html) and the [community model zoo](https://github.com/BVLC/caffe/wiki/Model-Zoo) -- [Installation instructions](http://caffe.berkeleyvision.org/installation.html) +[![Build status](https://ci.appveyor.com/api/projects/status/ew7cl2k1qfsnyql4/branch/windows?svg=true)](https://ci.appveyor.com/project/BVLC/caffe/branch/windows) AppVeyor (Windows build) -and step-by-step examples. +## Prebuilt binaries -[![Join the chat at https://gitter.im/BVLC/caffe](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/BVLC/caffe?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +Prebuilt binaries can be downloaded from the latest CI build on appveyor for the following configurations: -Please join the [caffe-users group](https://groups.google.com/forum/#!forum/caffe-users) or [gitter chat](https://gitter.im/BVLC/caffe) to ask questions and talk about methods and models. -Framework development discussions and thorough bug reports are collected on [Issues](https://github.com/BVLC/caffe/issues). +- Visual Studio 2015, CPU only, Python 3.5: [Caffe Release](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D14%2C%20WITH_NINJA%3D0%2C%20CMAKE_CONFIG%3DRelease%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D3%2C%20WITH_CUDA%3D0), ~~[Caffe Debug](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D14%2C%20WITH_NINJA%3D0%2C%20CMAKE_CONFIG%3DDebug%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D3%2C%20WITH_CUDA%3D0)~~ -Happy brewing! +- Visual Studio 2015, CUDA 8.0, Python 3.5: [Caffe Release](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D14%2C%20WITH_NINJA%3D1%2C%20CMAKE_CONFIG%3DRelease%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D3%2C%20WITH_CUDA%3D1) -## License and Citation +- Visual Studio 2015, CPU only, Python 2.7: [Caffe Release](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D14%2C%20WITH_NINJA%3D0%2C%20CMAKE_CONFIG%3DRelease%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D2%2C%20WITH_CUDA%3D0), [Caffe Debug](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D14%2C%20WITH_NINJA%3D0%2C%20CMAKE_CONFIG%3DDebug%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D2%2C%20WITH_CUDA%3D0) -Caffe is released under the [BSD 2-Clause license](https://github.com/BVLC/caffe/blob/master/LICENSE). -The BAIR/BVLC reference models are released for unrestricted use. +- Visual Studio 2015,CUDA 8.0, Python 2.7: [Caffe Release](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D14%2C%20WITH_NINJA%3D1%2C%20CMAKE_CONFIG%3DRelease%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D2%2C%20WITH_CUDA%3D1) -Please cite Caffe in your publications if it helps your research: +- Visual Studio 2013, CPU only, Python 2.7: [Caffe Release](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D12%2C%20WITH_NINJA%3D0%2C%20CMAKE_CONFIG%3DRelease%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D2%2C%20WITH_CUDA%3D0), [Caffe Debug](https://ci.appveyor.com/api/projects/BVLC/caffe/artifacts/build/caffe.zip?branch=windows&job=Environment%3A%20MSVC_VERSION%3D12%2C%20WITH_NINJA%3D0%2C%20CMAKE_CONFIG%3DDebug%2C%20CMAKE_BUILD_SHARED_LIBS%3D0%2C%20PYTHON_VERSION%3D2%2C%20WITH_CUDA%3D0) - @article{jia2014caffe, - Author = {Jia, Yangqing and Shelhamer, Evan and Donahue, Jeff and Karayev, Sergey and Long, Jonathan and Girshick, Ross and Guadarrama, Sergio and Darrell, Trevor}, - Journal = {arXiv preprint arXiv:1408.5093}, - Title = {Caffe: Convolutional Architecture for Fast Feature Embedding}, - Year = {2014} - } + +## Windows Setup + +### Requirements + + - Visual Studio 2013 or 2015 + - Technically only the VS C/C++ compiler is required (cl.exe) + - [CMake](https://cmake.org/) 3.4 or higher (Visual Studio and [Ninja](https://ninja-build.org/) generators are supported) + +### Optional Dependencies + + - Python for the pycaffe interface. Anaconda Python 2.7 or 3.5 x64 (or Miniconda) + - Matlab for the matcaffe interface. + - CUDA 7.5 or 8.0 (use CUDA 8 if using Visual Studio 2015) + - cuDNN v5 + + We assume that `cmake.exe` and `python.exe` are on your `PATH`. + +### Configuring and Building Caffe + +The fastest method to get started with caffe on Windows is by executing the following commands in a `cmd` prompt (we use `C:\Projects` as a root folder for the remainder of the instructions): +```cmd +C:\Projects> git clone https://github.com/BVLC/caffe.git +C:\Projects> cd caffe +C:\Projects\caffe> git checkout windows +:: Edit any of the options inside build_win.cmd to suit your needs +C:\Projects\caffe> scripts\build_win.cmd +``` +The `build_win.cmd` script will download the dependencies, create the Visual Studio project files (or the ninja build files) and build the Release configuration. By default all the required DLLs will be copied (or hard linked when possible) next to the consuming binaries. If you wish to disable this option, you can by changing the command line option `-DCOPY_PREREQUISITES=0`. The prebuilt libraries also provide a `prependpath.bat` batch script that can temporarily modify your `PATH` environment variable to make the required DLLs available. + +If you have GCC installed (e.g. through MinGW), then Ninja will detect it before detecting the Visual Studio compiler, causing errors. In this case you have several options: + +- [Pass CMake the path](https://cmake.org/Wiki/CMake_FAQ#How_do_I_use_a_different_compiler.3F) (Set `CMAKE_C_COMPILER=your/path/to/cl.exe` and `CMAKE_CXX_COMPILER=your/path/to/cl.exe`) +- or Use the Visual Studio Generator by setting `WITH_NINJA` to 0 (This is slower, but may work even if Ninja is failing.) +- or uninstall your copy of GCC + +The path to cl.exe is usually something like +`"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/your_processor_architecture/cl.exe".` +If you don't want to install Visual Studio, Microsoft's C/C++ compiler [can be obtained here](http://landinghub.visualstudio.com/visual-cpp-build-tools). + +Below is a more complete description of some of the steps involved in building caffe. + +### Install the caffe dependencies + +By default CMake will download and extract prebuilt dependencies for your compiler and python version. It will create a folder called `libraries` containing all the required dependencies inside your build folder. Alternatively you can build them yourself by following the instructions in the [caffe-builder](https://github.com/willyd/caffe-builder) [README](https://github.com/willyd/caffe-builder/blob/master/README.md). + +### Use cuDNN + +To use cuDNN the easiest way is to copy the content of the `cuda` folder into your CUDA toolkit installation directory. For example if you installed CUDA 8.0 and downloaded cudnn-8.0-windows10-x64-v5.1.zip you should copy the content of the `cuda` directory to `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v8.0`. Alternatively, you can define the CUDNN_ROOT cache variable to point to where you unpacked the cuDNN files e.g. `C:/Projects/caffe/cudnn-8.0-windows10-x64-v5.1/cuda`. For example the command in [scripts/build_win.cmd](scripts/build_win.cmd) would become: +``` +cmake -G"!CMAKE_GENERATOR!" ^ + -DBLAS=Open ^ + -DCMAKE_BUILD_TYPE:STRING=%CMAKE_CONFIG% ^ + -DBUILD_SHARED_LIBS:BOOL=%CMAKE_BUILD_SHARED_LIBS% ^ + -DBUILD_python:BOOL=%BUILD_PYTHON% ^ + -DBUILD_python_layer:BOOL=%BUILD_PYTHON_LAYER% ^ + -DBUILD_matlab:BOOL=%BUILD_MATLAB% ^ + -DCPU_ONLY:BOOL=%CPU_ONLY% ^ + -DCUDNN_ROOT=C:/Projects/caffe/cudnn-8.0-windows10-x64-v5.1/cuda ^ + -C "%cd%\libraries\caffe-builder-config.cmake" ^ + "%~dp0\.." +``` + +Alternatively, you can open `cmake-gui.exe` and set the variable from there and click `Generate`. + +### Building only for CPU + +If CUDA is not installed Caffe will default to a CPU_ONLY build. If you have CUDA installed but want a CPU only build you may use the CMake option `-DCPU_ONLY=1`. + +### Using the Python interface + +The recommended Python distribution is Anaconda or Miniconda. To successfully build the python interface you need to add the following conda channels: +``` +conda config --add channels conda-forge +conda config --add channels willyd +``` +and install the following packages: +``` +conda install --yes cmake ninja numpy scipy protobuf==3.1.0 six scikit-image pyyaml pydotplus graphviz +``` +If Python is installed the default is to build the python interface and python layers. If you wish to disable the python layers or the python build use the CMake options `-DBUILD_python_layer=0` and `-DBUILD_python=0` respectively. In order to use the python interface you need to either add the `C:\Projects\caffe\python` folder to your python path of copy the `C:\Projects\caffe\python\caffe` folder to your `site_packages` folder. + +### Using the MATLAB interface + +Follow the above procedure and use `-DBUILD_matlab=ON`. Change your current directory in MATLAB to `C:\Projects\caffe\matlab` and run the following command to run the tests: +``` +>> caffe.run_tests() +``` +If all tests pass you can test if the classification_demo works as well. First, from `C:\Projects\caffe` run `python scripts\download_model_binary.py models\bvlc_reference_caffenet` to download the pre-trained caffemodel from the model zoo. Then change your MATLAB directory to `C:\Projects\caffe\matlab\demo` and run `classification_demo`. + +### Using the Ninja generator + +You can choose to use the Ninja generator instead of Visual Studio for faster builds. To do so, change the option `set WITH_NINJA=1` in the `build_win.cmd` script. To install Ninja you can download the executable from github or install it via conda: +```cmd +> conda config --add channels conda-forge +> conda install ninja --yes +``` +When working with ninja you don't have the Visual Studio solutions as ninja is more akin to make. An alternative is to use [Visual Studio Code](https://code.visualstudio.com) with the CMake extensions and C++ extensions. + +### Building a shared library + +CMake can be used to build a shared library instead of the default static library. To do so follow the above procedure and use `-DBUILD_SHARED_LIBS=ON`. Please note however, that some tests (more specifically the solver related tests) will fail since both the test executable and caffe library do not share static objects contained in the protobuf library. + +### Troubleshooting + +Should you encounter any error please post the output of the above commands by redirecting the output to a file and open a topic on the [caffe-users list](https://groups.google.com/forum/#!forum/caffe-users) mailing list. + +## Known issues + +- The `GPUTimer` related test cases always fail on Windows. This seems to be a difference between UNIX and Windows. +- Shared library (DLL) build will have failing tests. +- Shared library build only works with the Ninja generator + +## Further Details + +Refer to the BVLC/caffe master branch README for all other details such as license, citation, and so on. diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000000..f37d4e6feb7 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,88 @@ +version: 1.0.{build} +clone_folder: c:\projects\caffe +environment: + matrix: + - MSVC_VERSION: 14 + WITH_NINJA: 0 + CMAKE_CONFIG: Release + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 3 + WITH_CUDA: 0 + + - MSVC_VERSION: 14 + WITH_NINJA: 0 + CMAKE_CONFIG: Release + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + + - MSVC_VERSION: 14 + WITH_NINJA: 0 + CMAKE_CONFIG: Debug + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + + - MSVC_VERSION: 14 + WITH_NINJA: 1 + CMAKE_CONFIG: Release + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + + - MSVC_VERSION: 14 + WITH_NINJA: 1 + CMAKE_CONFIG: Debug + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + + - MSVC_VERSION: 14 + WITH_NINJA: 1 + CMAKE_CONFIG: Release + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 1 + + - MSVC_VERSION: 14 + WITH_NINJA: 1 + CMAKE_CONFIG: Release + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 3 + WITH_CUDA: 1 + + - MSVC_VERSION: 12 + WITH_NINJA: 0 + CMAKE_CONFIG: Release + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + + - MSVC_VERSION: 12 + WITH_NINJA: 0 + CMAKE_CONFIG: Debug + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + + - MSVC_VERSION: 12 + WITH_NINJA: 1 + CMAKE_CONFIG: Release + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + + - MSVC_VERSION: 12 + WITH_NINJA: 1 + CMAKE_CONFIG: Debug + CMAKE_BUILD_SHARED_LIBS: 0 + PYTHON_VERSION: 2 + WITH_CUDA: 0 + +build_script: +- cmd: >- + call scripts\build_win.cmd + +artifacts: + - path: build\install + name: caffe \ No newline at end of file diff --git a/cmake/CaffeGetPrerequisites.cmake b/cmake/CaffeGetPrerequisites.cmake new file mode 100644 index 00000000000..bf5bc271436 --- /dev/null +++ b/cmake/CaffeGetPrerequisites.cmake @@ -0,0 +1,1036 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#.rst: +# GetPrerequisites +# ---------------- +# +# Functions to analyze and list executable file prerequisites. +# +# This module provides functions to list the .dll, .dylib or .so files +# that an executable or shared library file depends on. (Its +# prerequisites.) +# +# It uses various tools to obtain the list of required shared library +# files: +# +# :: +# +# dumpbin (Windows) +# objdump (MinGW on Windows) +# ldd (Linux/Unix) +# otool (Mac OSX) +# +# The following functions are provided by this module: +# +# :: +# +# get_prerequisites +# list_prerequisites +# list_prerequisites_by_glob +# gp_append_unique +# is_file_executable +# gp_item_default_embedded_path +# (projects can override with gp_item_default_embedded_path_override) +# gp_resolve_item +# (projects can override with gp_resolve_item_override) +# gp_resolved_file_type +# (projects can override with gp_resolved_file_type_override) +# gp_file_type +# +# Requires CMake 2.6 or greater because it uses function, break, return +# and PARENT_SCOPE. +# +# :: +# +# GET_PREREQUISITES( +# []) +# +# Get the list of shared library files required by . The list +# in the variable named should be empty on first +# entry to this function. On exit, will contain the +# list of required shared library files. +# +# is the full path to an executable file. +# is the name of a CMake variable to contain the results. +# must be 0 or 1 indicating whether to include or +# exclude "system" prerequisites. If is set to 1 all +# prerequisites will be found recursively, if set to 0 only direct +# prerequisites are listed. is the path to the top level +# executable used for @executable_path replacment on the Mac. is +# a list of paths where libraries might be found: these paths are +# searched first when a target without any path info is given. Then +# standard system locations are also searched: PATH, Framework +# locations, /usr/lib... +# +# :: +# +# LIST_PREREQUISITES( [ [ []]]) +# +# Print a message listing the prerequisites of . +# +# is the name of a shared library or executable target or the +# full path to a shared library or executable file. If is set +# to 1 all prerequisites will be found recursively, if set to 0 only +# direct prerequisites are listed. must be 0 or 1 +# indicating whether to include or exclude "system" prerequisites. With +# set to 0 only the full path names of the prerequisites are +# printed, set to 1 extra informatin will be displayed. +# +# :: +# +# LIST_PREREQUISITES_BY_GLOB( ) +# +# Print the prerequisites of shared library and executable files +# matching a globbing pattern. is GLOB or GLOB_RECURSE and +# is a globbing expression used with "file(GLOB" or +# "file(GLOB_RECURSE" to retrieve a list of matching files. If a +# matching file is executable, its prerequisites are listed. +# +# Any additional (optional) arguments provided are passed along as the +# optional arguments to the list_prerequisites calls. +# +# :: +# +# GP_APPEND_UNIQUE( ) +# +# Append to the list variable only if the value is +# not already in the list. +# +# :: +# +# IS_FILE_EXECUTABLE( ) +# +# Return 1 in if is a binary executable, 0 +# otherwise. +# +# :: +# +# GP_ITEM_DEFAULT_EMBEDDED_PATH( ) +# +# Return the path that others should refer to the item by when the item +# is embedded inside a bundle. +# +# Override on a per-project basis by providing a project-specific +# gp_item_default_embedded_path_override function. +# +# :: +# +# GP_RESOLVE_ITEM( +# []) +# +# Resolve an item into an existing full path file. +# +# Override on a per-project basis by providing a project-specific +# gp_resolve_item_override function. +# +# :: +# +# GP_RESOLVED_FILE_TYPE( +# []) +# +# Return the type of with respect to . String +# describing type of prerequisite is returned in variable named +# . +# +# Use and if necessary to resolve non-absolute +# values -- but only for non-embedded items. +# +# Possible types are: +# +# :: +# +# system +# local +# embedded +# other +# +# Override on a per-project basis by providing a project-specific +# gp_resolved_file_type_override function. +# +# :: +# +# GP_FILE_TYPE( ) +# +# Return the type of with respect to . String +# describing type of prerequisite is returned in variable named +# . +# +# Possible types are: +# +# :: +# +# system +# local +# embedded +# other + +function(gp_append_unique list_var value) + set(contains 0) + + foreach(item ${${list_var}}) + if(item STREQUAL "${value}") + set(contains 1) + break() + endif() + endforeach() + + if(NOT contains) + set(${list_var} ${${list_var}} "${value}" PARENT_SCOPE) + endif() +endfunction() + + +function(is_file_executable file result_var) + # + # A file is not executable until proven otherwise: + # + set(${result_var} 0 PARENT_SCOPE) + + get_filename_component(file_full "${file}" ABSOLUTE) + string(TOLOWER "${file_full}" file_full_lower) + + # If file name ends in .exe on Windows, *assume* executable: + # + if(WIN32 AND NOT UNIX) + if("${file_full_lower}" MATCHES "\\.exe$") + set(${result_var} 1 PARENT_SCOPE) + return() + endif() + + # A clause could be added here that uses output or return value of dumpbin + # to determine ${result_var}. In 99%+? practical cases, the exe name + # match will be sufficient... + # + endif() + + # Use the information returned from the Unix shell command "file" to + # determine if ${file_full} should be considered an executable file... + # + # If the file command's output contains "executable" and does *not* contain + # "text" then it is likely an executable suitable for prerequisite analysis + # via the get_prerequisites macro. + # + if(UNIX) + if(NOT file_cmd) + find_program(file_cmd "file") + mark_as_advanced(file_cmd) + endif() + + if(file_cmd) + execute_process(COMMAND "${file_cmd}" "${file_full}" + RESULT_VARIABLE file_rv + OUTPUT_VARIABLE file_ov + ERROR_VARIABLE file_ev + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT file_rv STREQUAL "0") + message(FATAL_ERROR "${file_cmd} failed: ${file_rv}\n${file_ev}") + endif() + + # Replace the name of the file in the output with a placeholder token + # (the string " _file_full_ ") so that just in case the path name of + # the file contains the word "text" or "executable" we are not fooled + # into thinking "the wrong thing" because the file name matches the + # other 'file' command output we are looking for... + # + string(REPLACE "${file_full}" " _file_full_ " file_ov "${file_ov}") + string(TOLOWER "${file_ov}" file_ov) + + #message(STATUS "file_ov='${file_ov}'") + if("${file_ov}" MATCHES "executable") + #message(STATUS "executable!") + if("${file_ov}" MATCHES "text") + #message(STATUS "but text, so *not* a binary executable!") + else() + set(${result_var} 1 PARENT_SCOPE) + return() + endif() + endif() + + # Also detect position independent executables on Linux, + # where "file" gives "shared object ... (uses shared libraries)" + if("${file_ov}" MATCHES "shared object.*\(uses shared libs\)") + set(${result_var} 1 PARENT_SCOPE) + return() + endif() + + # "file" version 5.22 does not print "(used shared libraries)" + # but uses "interpreter" + if("${file_ov}" MATCHES "shared object.*interpreter") + set(${result_var} 1 PARENT_SCOPE) + return() + endif() + + else() + message(STATUS "warning: No 'file' command, skipping execute_process...") + endif() + endif() +endfunction() + + +function(gp_item_default_embedded_path item default_embedded_path_var) + + # On Windows and Linux, "embed" prerequisites in the same directory + # as the executable by default: + # + set(path "@executable_path") + set(overridden 0) + + # On the Mac, relative to the executable depending on the type + # of the thing we are embedding: + # + if(APPLE) + # + # The assumption here is that all executables in the bundle will be + # in same-level-directories inside the bundle. The parent directory + # of an executable inside the bundle should be MacOS or a sibling of + # MacOS and all embedded paths returned from here will begin with + # "@executable_path/../" and will work from all executables in all + # such same-level-directories inside the bundle. + # + + # By default, embed things right next to the main bundle executable: + # + set(path "@executable_path/../../Contents/MacOS") + + # Embed .dylibs right next to the main bundle executable: + # + if(item MATCHES "\\.dylib$") + set(path "@executable_path/../MacOS") + set(overridden 1) + endif() + + # Embed frameworks in the embedded "Frameworks" directory (sibling of MacOS): + # + if(NOT overridden) + if(item MATCHES "[^/]+\\.framework/") + set(path "@executable_path/../Frameworks") + set(overridden 1) + endif() + endif() + endif() + + # Provide a hook so that projects can override the default embedded location + # of any given library by whatever logic they choose: + # + if(COMMAND gp_item_default_embedded_path_override) + gp_item_default_embedded_path_override("${item}" path) + endif() + + set(${default_embedded_path_var} "${path}" PARENT_SCOPE) +endfunction() + + +function(gp_resolve_item context item exepath dirs resolved_item_var) + set(resolved 0) + set(resolved_item "${item}") + if(ARGC GREATER 5) + set(rpaths "${ARGV5}") + else() + set(rpaths "") + endif() + + # Is it already resolved? + # + if(IS_ABSOLUTE "${resolved_item}" AND EXISTS "${resolved_item}") + set(resolved 1) + endif() + + if(NOT resolved) + if(item MATCHES "^@executable_path") + # + # @executable_path references are assumed relative to exepath + # + string(REPLACE "@executable_path" "${exepath}" ri "${item}") + get_filename_component(ri "${ri}" ABSOLUTE) + + if(EXISTS "${ri}") + #message(STATUS "info: embedded item exists (${ri})") + set(resolved 1) + set(resolved_item "${ri}") + else() + message(STATUS "warning: embedded item does not exist '${ri}'") + endif() + endif() + endif() + + if(NOT resolved) + if(item MATCHES "^@loader_path") + # + # @loader_path references are assumed relative to the + # PATH of the given "context" (presumably another library) + # + get_filename_component(contextpath "${context}" PATH) + string(REPLACE "@loader_path" "${contextpath}" ri "${item}") + get_filename_component(ri "${ri}" ABSOLUTE) + + if(EXISTS "${ri}") + #message(STATUS "info: embedded item exists (${ri})") + set(resolved 1) + set(resolved_item "${ri}") + else() + message(STATUS "warning: embedded item does not exist '${ri}'") + endif() + endif() + endif() + + if(NOT resolved) + if(item MATCHES "^@rpath") + # + # @rpath references are relative to the paths built into the binaries with -rpath + # We handle this case like we do for other Unixes + # + string(REPLACE "@rpath/" "" norpath_item "${item}") + + set(ri "ri-NOTFOUND") + find_file(ri "${norpath_item}" ${exepath} ${dirs} ${rpaths} NO_DEFAULT_PATH) + if(ri) + #message(STATUS "info: 'find_file' in exepath/dirs/rpaths (${ri})") + set(resolved 1) + set(resolved_item "${ri}") + set(ri "ri-NOTFOUND") + endif() + + endif() + endif() + + if(NOT resolved) + set(ri "ri-NOTFOUND") + find_file(ri "${item}" ${exepath} ${dirs} NO_DEFAULT_PATH) + find_file(ri "${item}" ${exepath} ${dirs} /usr/lib) + if(ri) + #message(STATUS "info: 'find_file' in exepath/dirs (${ri})") + set(resolved 1) + set(resolved_item "${ri}") + set(ri "ri-NOTFOUND") + endif() + endif() + + if(NOT resolved) + if(item MATCHES "[^/]+\\.framework/") + set(fw "fw-NOTFOUND") + find_file(fw "${item}" + "~/Library/Frameworks" + "/Library/Frameworks" + "/System/Library/Frameworks" + ) + if(fw) + #message(STATUS "info: 'find_file' found framework (${fw})") + set(resolved 1) + set(resolved_item "${fw}") + set(fw "fw-NOTFOUND") + endif() + endif() + endif() + + # Using find_program on Windows will find dll files that are in the PATH. + # (Converting simple file names into full path names if found.) + # + if(WIN32 AND NOT UNIX) + if(NOT resolved) + set(ri "ri-NOTFOUND") + find_program(ri "${item}" PATHS ${exepath} ${dirs} NO_DEFAULT_PATH) + find_program(ri "${item}" PATHS ${exepath} ${dirs}) + if(ri) + #message(STATUS "info: 'find_program' in exepath/dirs (${ri})") + set(resolved 1) + set(resolved_item "${ri}") + set(ri "ri-NOTFOUND") + endif() + endif() + endif() + + # Provide a hook so that projects can override item resolution + # by whatever logic they choose: + # + if(COMMAND gp_resolve_item_override) + gp_resolve_item_override("${context}" "${item}" "${exepath}" "${dirs}" resolved_item resolved) + endif() + + if(NOT resolved) + message(STATUS " +warning: cannot resolve item '${item}' + + possible problems: + need more directories? + need to use InstallRequiredSystemLibraries? + run in install tree instead of build tree? +") +# message(STATUS " +#****************************************************************************** +#warning: cannot resolve item '${item}' +# +# possible problems: +# need more directories? +# need to use InstallRequiredSystemLibraries? +# run in install tree instead of build tree? +# +# context='${context}' +# item='${item}' +# exepath='${exepath}' +# dirs='${dirs}' +# resolved_item_var='${resolved_item_var}' +#****************************************************************************** +#") + endif() + + set(${resolved_item_var} "${resolved_item}" PARENT_SCOPE) +endfunction() + + +function(gp_resolved_file_type original_file file exepath dirs type_var) + if(ARGC GREATER 5) + set(rpaths "${ARGV5}") + else() + set(rpaths "") + endif() + #message(STATUS "**") + + if(NOT IS_ABSOLUTE "${original_file}") + message(STATUS "warning: gp_resolved_file_type expects absolute full path for first arg original_file") + endif() + if(IS_ABSOLUTE "${original_file}") + get_filename_component(original_file "${original_file}" ABSOLUTE) # canonicalize path + endif() + + set(is_embedded 0) + set(is_local 0) + set(is_system 0) + + set(resolved_file "${file}") + + if("${file}" MATCHES "^@(executable|loader)_path") + set(is_embedded 1) + endif() + + if(NOT is_embedded) + if(NOT IS_ABSOLUTE "${file}") + gp_resolve_item("${original_file}" "${file}" "${exepath}" "${dirs}" resolved_file "${rpaths}") + endif() + if(IS_ABSOLUTE "${resolved_file}") + get_filename_component(resolved_file "${resolved_file}" ABSOLUTE) # canonicalize path + endif() + + string(TOLOWER "${original_file}" original_lower) + string(TOLOWER "${resolved_file}" lower) + + if(UNIX) + if(resolved_file MATCHES "^(/lib/|/lib32/|/lib64/|/usr/lib/|/usr/lib32/|/usr/lib64/|/usr/X11R6/|/usr/bin/)") + set(is_system 1) + endif() + endif() + + if(APPLE) + if(resolved_file MATCHES "^(/System/Library/|/usr/lib/)") + set(is_system 1) + endif() + endif() + + if(WIN32) + string(TOLOWER "$ENV{SystemRoot}" sysroot) + file(TO_CMAKE_PATH "${sysroot}" sysroot) + + string(TOLOWER "$ENV{windir}" windir) + file(TO_CMAKE_PATH "${windir}" windir) + + if(lower MATCHES "^(${sysroot}/sys(tem|wow)|${windir}/sys(tem|wow)|(.*/)*(msvc|api-ms-win-)[^/]+dll)") + set(is_system 1) + endif() + + if(UNIX) + # if cygwin, we can get the properly formed windows paths from cygpath + find_program(CYGPATH_EXECUTABLE cygpath) + + if(CYGPATH_EXECUTABLE) + execute_process(COMMAND ${CYGPATH_EXECUTABLE} -W + RESULT_VARIABLE env_rv + OUTPUT_VARIABLE env_windir + ERROR_VARIABLE env_ev + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT env_rv STREQUAL "0") + message(FATAL_ERROR "${CYGPATH_EXECUTABLE} -W failed: ${env_rv}\n${env_ev}") + endif() + execute_process(COMMAND ${CYGPATH_EXECUTABLE} -S + RESULT_VARIABLE env_rv + OUTPUT_VARIABLE env_sysdir + ERROR_VARIABLE env_ev + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT env_rv STREQUAL "0") + message(FATAL_ERROR "${CYGPATH_EXECUTABLE} -S failed: ${env_rv}\n${env_ev}") + endif() + string(TOLOWER "${env_windir}" windir) + string(TOLOWER "${env_sysdir}" sysroot) + + if(lower MATCHES "^(${sysroot}/sys(tem|wow)|${windir}/sys(tem|wow)|(.*/)*(msvc|api-ms-win-)[^/]+dll)") + set(is_system 1) + endif() + endif() + endif() + endif() + + if(NOT is_system) + get_filename_component(original_path "${original_lower}" PATH) + get_filename_component(path "${lower}" PATH) + if(original_path STREQUAL path) + set(is_local 1) + else() + string(LENGTH "${original_path}/" original_length) + string(LENGTH "${lower}" path_length) + if(${path_length} GREATER ${original_length}) + string(SUBSTRING "${lower}" 0 ${original_length} path) + if("${original_path}/" STREQUAL path) + set(is_embedded 1) + endif() + endif() + endif() + endif() + endif() + + # Return type string based on computed booleans: + # + set(type "other") + + if(is_system) + set(type "system") + elseif(is_embedded) + set(type "embedded") + elseif(is_local) + set(type "local") + endif() + + #message(STATUS "gp_resolved_file_type: '${file}' '${resolved_file}'") + #message(STATUS " type: '${type}'") + + if(NOT is_embedded) + if(NOT IS_ABSOLUTE "${resolved_file}") + if(lower MATCHES "^msvc[^/]+dll" AND is_system) + message(STATUS "info: non-absolute msvc file '${file}' returning type '${type}'") + else() + message(STATUS "warning: gp_resolved_file_type non-absolute file '${file}' returning type '${type}' -- possibly incorrect") + endif() + endif() + endif() + + # Provide a hook so that projects can override the decision on whether a + # library belongs to the system or not by whatever logic they choose: + # + if(COMMAND gp_resolved_file_type_override) + gp_resolved_file_type_override("${resolved_file}" type) + endif() + + set(${type_var} "${type}" PARENT_SCOPE) + + #message(STATUS "**") +endfunction() + + +function(gp_file_type original_file file type_var) + if(NOT IS_ABSOLUTE "${original_file}") + message(STATUS "warning: gp_file_type expects absolute full path for first arg original_file") + endif() + + get_filename_component(exepath "${original_file}" PATH) + + set(type "") + gp_resolved_file_type("${original_file}" "${file}" "${exepath}" "" type) + + set(${type_var} "${type}" PARENT_SCOPE) +endfunction() + + +function(get_prerequisites target prerequisites_var exclude_system recurse exepath dirs) + set(verbose 0) + set(eol_char "E") + if(ARGC GREATER 6) + set(rpaths "${ARGV6}") + else() + set(rpaths "") + endif() + + if(NOT IS_ABSOLUTE "${target}") + message("warning: target '${target}' is not absolute...") + endif() + + if(NOT EXISTS "${target}") + message("warning: target '${target}' does not exist...") + set(${prerequisites_var} "" PARENT_SCOPE) + return() + endif() + + set(gp_cmd_paths ${gp_cmd_paths} + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\14.0;InstallDir]/../../VC/bin" + "$ENV{VS140COMNTOOLS}/../../VC/bin" + "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\12.0;InstallDir]/../../VC/bin" + "$ENV{VS120COMNTOOLS}/../../VC/bin" + "C:/Program Files (x86)/Microsoft Visual Studio 12.0/VC/bin" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\11.0;InstallDir]/../../VC/bin" + "$ENV{VS110COMNTOOLS}/../../VC/bin" + "C:/Program Files (x86)/Microsoft Visual Studio 11.0/VC/bin" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\10.0;InstallDir]/../../VC/bin" + "$ENV{VS100COMNTOOLS}/../../VC/bin" + "C:/Program Files (x86)/Microsoft Visual Studio 10.0/VC/bin" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\9.0;InstallDir]/../../VC/bin" + "$ENV{VS90COMNTOOLS}/../../VC/bin" + "C:/Program Files/Microsoft Visual Studio 9.0/VC/bin" + "C:/Program Files (x86)/Microsoft Visual Studio 9.0/VC/bin" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\8.0;InstallDir]/../../VC/bin" + "$ENV{VS80COMNTOOLS}/../../VC/bin" + "C:/Program Files/Microsoft Visual Studio 8/VC/BIN" + "C:/Program Files (x86)/Microsoft Visual Studio 8/VC/BIN" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\7.1;InstallDir]/../../VC7/bin" + "$ENV{VS71COMNTOOLS}/../../VC7/bin" + "C:/Program Files/Microsoft Visual Studio .NET 2003/VC7/BIN" + "C:/Program Files (x86)/Microsoft Visual Studio .NET 2003/VC7/BIN" + "/usr/local/bin" + "/usr/bin" + ) + + # + # + # Try to choose the right tool by default. Caller can set gp_tool prior to + # calling this function to force using a different tool. + # + if(NOT gp_tool) + set(gp_tool "ldd") + + if(APPLE) + set(gp_tool "otool") + endif() + + if(WIN32 AND NOT UNIX) # This is how to check for cygwin, har! + find_program(gp_dumpbin "dumpbin" PATHS ${gp_cmd_paths}) + if(gp_dumpbin) + set(gp_tool "dumpbin") + else() # Try harder. Maybe we're on MinGW + set(gp_tool "objdump") + endif() + endif() + endif() + + find_program(gp_cmd ${gp_tool} PATHS ${gp_cmd_paths}) + + if(NOT gp_cmd) + message(STATUS "warning: could not find '${gp_tool}' - cannot analyze prerequisites...") + return() + endif() + + set(gp_cmd_maybe_filter) # optional command to pre-filter gp_tool results + + if(gp_tool STREQUAL "ldd") + set(gp_cmd_args "") + set(gp_regex "^[\t ]*[^\t ]+ => ([^\t\(]+) .*${eol_char}$") + set(gp_regex_error "not found${eol_char}$") + set(gp_regex_fallback "^[\t ]*([^\t ]+) => ([^\t ]+).*${eol_char}$") + set(gp_regex_cmp_count 1) + elseif(gp_tool STREQUAL "otool") + set(gp_cmd_args "-L") + set(gp_regex "^\t([^\t]+) \\(compatibility version ([0-9]+.[0-9]+.[0-9]+), current version ([0-9]+.[0-9]+.[0-9]+)\\)${eol_char}$") + set(gp_regex_error "") + set(gp_regex_fallback "") + set(gp_regex_cmp_count 3) + elseif(gp_tool STREQUAL "dumpbin") + set(gp_cmd_args "/dependents") + set(gp_regex "^ ([^ ].*[Dd][Ll][Ll])${eol_char}$") + set(gp_regex_error "") + set(gp_regex_fallback "") + set(gp_regex_cmp_count 1) + elseif(gp_tool STREQUAL "objdump") + set(gp_cmd_args "-p") + set(gp_regex "^\t*DLL Name: (.*\\.[Dd][Ll][Ll])${eol_char}$") + set(gp_regex_error "") + set(gp_regex_fallback "") + set(gp_regex_cmp_count 1) + # objdump generates copious output so we create a grep filter to pre-filter results + if(WIN32) + find_program(gp_grep_cmd findstr) + else() + find_program(gp_grep_cmd grep) + endif() + if(gp_grep_cmd) + set(gp_cmd_maybe_filter COMMAND ${gp_grep_cmd} "-a" "^[[:blank:]]*DLL Name: ") + endif() + else() + message(STATUS "warning: gp_tool='${gp_tool}' is an unknown tool...") + message(STATUS "CMake function get_prerequisites needs more code to handle '${gp_tool}'") + message(STATUS "Valid gp_tool values are dumpbin, ldd, objdump and otool.") + return() + endif() + + + if(gp_tool STREQUAL "dumpbin") + # When running dumpbin, it also needs the "Common7/IDE" directory in the + # PATH. It will already be in the PATH if being run from a Visual Studio + # command prompt. Add it to the PATH here in case we are running from a + # different command prompt. + # + get_filename_component(gp_cmd_dir "${gp_cmd}" PATH) + get_filename_component(gp_cmd_dlls_dir "${gp_cmd_dir}/../../Common7/IDE" ABSOLUTE) + # Use cmake paths as a user may have a PATH element ending with a backslash. + # This will escape the list delimiter and create havoc! + if(EXISTS "${gp_cmd_dlls_dir}") + # only add to the path if it is not already in the path + set(gp_found_cmd_dlls_dir 0) + file(TO_CMAKE_PATH "$ENV{PATH}" env_path) + foreach(gp_env_path_element ${env_path}) + if(gp_env_path_element STREQUAL gp_cmd_dlls_dir) + set(gp_found_cmd_dlls_dir 1) + endif() + endforeach() + + if(NOT gp_found_cmd_dlls_dir) + file(TO_NATIVE_PATH "${gp_cmd_dlls_dir}" gp_cmd_dlls_dir) + set(ENV{PATH} "$ENV{PATH};${gp_cmd_dlls_dir}") + endif() + endif() + endif() + # + # + + if(gp_tool STREQUAL "ldd") + set(old_ld_env "$ENV{LD_LIBRARY_PATH}") + set(new_ld_env "${exepath}") + foreach(dir ${dirs}) + string(APPEND new_ld_env ":${dir}") + endforeach() + set(ENV{LD_LIBRARY_PATH} "${new_ld_env}:$ENV{LD_LIBRARY_PATH}") + endif() + + + # Track new prerequisites at each new level of recursion. Start with an + # empty list at each level: + # + set(unseen_prereqs) + + # Run gp_cmd on the target: + # + execute_process( + COMMAND ${gp_cmd} ${gp_cmd_args} ${target} + ${gp_cmd_maybe_filter} + RESULT_VARIABLE gp_rv + OUTPUT_VARIABLE gp_cmd_ov + ERROR_VARIABLE gp_ev + ) + + if(gp_tool STREQUAL "dumpbin") + # Exclude delay load dependencies under windows (they are listed in dumpbin output after the message below) + string(FIND "${gp_cmd_ov}" "Image has the following delay load dependencies" gp_delayload_pos) + if (${gp_delayload_pos} GREATER -1) + string(SUBSTRING "${gp_cmd_ov}" 0 ${gp_delayload_pos} gp_cmd_ov_no_delayload_deps) + string(SUBSTRING "${gp_cmd_ov}" ${gp_delayload_pos} -1 gp_cmd_ov_delayload_deps) + if (verbose) + message(STATUS "GetPrequisites(${target}) : ignoring the following delay load dependencies :\n ${gp_cmd_ov_delayload_deps}") + endif() + set(gp_cmd_ov ${gp_cmd_ov_no_delayload_deps}) + endif() + endif() + + if(NOT gp_rv STREQUAL "0") + if(gp_tool STREQUAL "dumpbin") + # dumpbin error messages seem to go to stdout + message(FATAL_ERROR "${gp_cmd} failed: ${gp_rv}\n${gp_ev}\n${gp_cmd_ov}") + else() + message(FATAL_ERROR "${gp_cmd} failed: ${gp_rv}\n${gp_ev}") + endif() + endif() + + if(gp_tool STREQUAL "ldd") + set(ENV{LD_LIBRARY_PATH} "${old_ld_env}") + endif() + + if(verbose) + message(STATUS "") + message(STATUS "gp_cmd_ov='${gp_cmd_ov}'") + message(STATUS "") + endif() + + get_filename_component(target_dir "${target}" PATH) + + # Convert to a list of lines: + # + string(REPLACE ";" "\\;" candidates "${gp_cmd_ov}") + string(REPLACE "\n" "${eol_char};" candidates "${candidates}") + + # check for install id and remove it from list, since otool -L can include a + # reference to itself + set(gp_install_id) + if(gp_tool STREQUAL "otool") + execute_process( + COMMAND otool -D ${target} + RESULT_VARIABLE otool_rv + OUTPUT_VARIABLE gp_install_id_ov + ERROR_VARIABLE otool_ev + ) + if(NOT otool_rv STREQUAL "0") + message(FATAL_ERROR "otool -D failed: ${otool_rv}\n${otool_ev}") + endif() + # second line is install name + string(REGEX REPLACE ".*:\n" "" gp_install_id "${gp_install_id_ov}") + if(gp_install_id) + # trim + string(REGEX MATCH "[^\n ].*[^\n ]" gp_install_id "${gp_install_id}") + #message("INSTALL ID is \"${gp_install_id}\"") + endif() + endif() + + # Analyze each line for file names that match the regular expression: + # + foreach(candidate ${candidates}) + if("${candidate}" MATCHES "${gp_regex}") + + # Extract information from each candidate: + if(gp_regex_error AND "${candidate}" MATCHES "${gp_regex_error}") + string(REGEX REPLACE "${gp_regex_fallback}" "\\1" raw_item "${candidate}") + else() + string(REGEX REPLACE "${gp_regex}" "\\1" raw_item "${candidate}") + endif() + + if(gp_regex_cmp_count GREATER 1) + string(REGEX REPLACE "${gp_regex}" "\\2" raw_compat_version "${candidate}") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$" "\\1" compat_major_version "${raw_compat_version}") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$" "\\2" compat_minor_version "${raw_compat_version}") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$" "\\3" compat_patch_version "${raw_compat_version}") + endif() + + if(gp_regex_cmp_count GREATER 2) + string(REGEX REPLACE "${gp_regex}" "\\3" raw_current_version "${candidate}") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$" "\\1" current_major_version "${raw_current_version}") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$" "\\2" current_minor_version "${raw_current_version}") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$" "\\3" current_patch_version "${raw_current_version}") + endif() + + # Use the raw_item as the list entries returned by this function. Use the + # gp_resolve_item function to resolve it to an actual full path file if + # necessary. + # + set(item "${raw_item}") + + # Add each item unless it is excluded: + # + set(add_item 1) + + if(item STREQUAL gp_install_id) + set(add_item 0) + endif() + + if(add_item AND ${exclude_system}) + set(type "") + gp_resolved_file_type("${target}" "${item}" "${exepath}" "${dirs}" type "${rpaths}") + + if(type STREQUAL "system") + set(add_item 0) + endif() + endif() + + if(add_item) + list(LENGTH ${prerequisites_var} list_length_before_append) + gp_append_unique(${prerequisites_var} "${item}") + list(LENGTH ${prerequisites_var} list_length_after_append) + + if(${recurse}) + # If item was really added, this is the first time we have seen it. + # Add it to unseen_prereqs so that we can recursively add *its* + # prerequisites... + # + # But first: resolve its name to an absolute full path name such + # that the analysis tools can simply accept it as input. + # + if(NOT list_length_before_append EQUAL list_length_after_append) + gp_resolve_item("${target}" "${item}" "${exepath}" "${dirs}" resolved_item "${rpaths}") + if(EXISTS ${resolved_item}) + # Recurse only if we could resolve the item. + # Otherwise the prerequisites_var list will be cleared + set(unseen_prereqs ${unseen_prereqs} "${resolved_item}") + endif() + endif() + endif() + endif() + else() + if(verbose) + message(STATUS "ignoring non-matching line: '${candidate}'") + endif() + endif() + endforeach() + + list(LENGTH ${prerequisites_var} prerequisites_var_length) + if(prerequisites_var_length GREATER 0) + list(SORT ${prerequisites_var}) + endif() + if(${recurse}) + set(more_inputs ${unseen_prereqs}) + foreach(input ${more_inputs}) + get_prerequisites("${input}" ${prerequisites_var} ${exclude_system} ${recurse} "${exepath}" "${dirs}" "${rpaths}") + endforeach() + endif() + + set(${prerequisites_var} ${${prerequisites_var}} PARENT_SCOPE) +endfunction() + + +function(list_prerequisites target) + if(ARGC GREATER 1 AND NOT "${ARGV1}" STREQUAL "") + set(all "${ARGV1}") + else() + set(all 1) + endif() + + if(ARGC GREATER 2 AND NOT "${ARGV2}" STREQUAL "") + set(exclude_system "${ARGV2}") + else() + set(exclude_system 0) + endif() + + if(ARGC GREATER 3 AND NOT "${ARGV3}" STREQUAL "") + set(verbose "${ARGV3}") + else() + set(verbose 0) + endif() + + set(count 0) + set(count_str "") + set(print_count "${verbose}") + set(print_prerequisite_type "${verbose}") + set(print_target "${verbose}") + set(type_str "") + + get_filename_component(exepath "${target}" PATH) + + set(prereqs "") + get_prerequisites("${target}" prereqs ${exclude_system} ${all} "${exepath}" "") + + if(print_target) + message(STATUS "File '${target}' depends on:") + endif() + + foreach(d ${prereqs}) + math(EXPR count "${count} + 1") + + if(print_count) + set(count_str "${count}. ") + endif() + + if(print_prerequisite_type) + gp_file_type("${target}" "${d}" type) + set(type_str " (${type})") + endif() + + message(STATUS "${count_str}${d}${type_str}") + endforeach() +endfunction() + + +function(list_prerequisites_by_glob glob_arg glob_exp) + message(STATUS "=============================================================================") + message(STATUS "List prerequisites of executables matching ${glob_arg} '${glob_exp}'") + message(STATUS "") + file(${glob_arg} file_list ${glob_exp}) + foreach(f ${file_list}) + is_file_executable("${f}" is_f_executable) + if(is_f_executable) + message(STATUS "=============================================================================") + list_prerequisites("${f}" ${ARGN}) + message(STATUS "") + endif() + endforeach() +endfunction() diff --git a/cmake/ConfigGen.cmake b/cmake/ConfigGen.cmake index 09bb09b4ff2..946a3ee98bf 100644 --- a/cmake/ConfigGen.cmake +++ b/cmake/ConfigGen.cmake @@ -24,6 +24,50 @@ function(caffe_generate_export_configs) set(HAVE_CUDA FALSE) endif() + set(GFLAGS_IMPORTED OFF) + foreach(_lib ${GFLAGS_LIBRARIES}) + if(TARGET ${_lib}) + set(GFLAGS_IMPORTED ON) + endif() + endforeach() + + set(GLOG_IMPORTED OFF) + foreach(_lib ${GLOG_LIBRARIES}) + if(TARGET ${_lib}) + set(GLOG_IMPORTED ON) + endif() + endforeach() + + set(HDF5_IMPORTED OFF) + foreach(_lib ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) + if(TARGET ${_lib}) + set(HDF5_IMPORTED ON) + endif() + endforeach() + + set(LMDB_IMPORTED OFF) + if(USE_LMDB) + foreach(_lib ${LMDB_LIBRARIES}) + if(TARGET ${_lib}) + set(LMDB_IMPORTED ON) + endif() + endforeach() + endif() + set(LEVELDB_IMPORTED OFF) + set(SNAPPY_IMPORTED OFF) + if(USE_LEVELDB) + foreach(_lib ${LevelDB_LIBRARIES}) + if(TARGET ${_lib}) + set(LEVELDB_IMPORTED ON) + endif() + endforeach() + foreach(_lib ${Snappy_LIBRARIES}) + if(TARGET ${_lib}) + set(SNAPPY_IMPORTED ON) + endif() + endforeach() + endif() + if(NOT HAVE_CUDNN) set(HAVE_CUDNN FALSE) endif() diff --git a/cmake/Cuda.cmake b/cmake/Cuda.cmake index b2b19e8b669..9ce2410979e 100644 --- a/cmake/Cuda.cmake +++ b/cmake/Cuda.cmake @@ -36,8 +36,12 @@ function(caffe_detect_installed_gpus out_variable) ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) if(__nvcc_res EQUAL 0) + # nvcc outputs text containing line breaks when building with MSVC. + # The line below prevents CMake from inserting a variable with line + # breaks in the cache + string(REGEX MATCH "([1-9].[0-9])" __nvcc_out "${__nvcc_out}") string(REPLACE "2.1" "2.1(2.0)" __nvcc_out "${__nvcc_out}") - set(CUDA_gpu_detect_output ${__nvcc_out} CACHE INTERNAL "Returned GPU architetures from caffe_detect_gpus tool" FORCE) + set(CUDA_gpu_detect_output ${__nvcc_out} CACHE INTERNAL "Returned GPU architetures from caffe_detect_gpus tool" FORCE) endif() endif() @@ -174,11 +178,21 @@ function(detect_cuDNN) find_path(CUDNN_INCLUDE cudnn.h PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT} ${CUDA_TOOLKIT_INCLUDE} + PATH_SUFFIXES include DOC "Path to cuDNN include directory." ) + + unset(_path_suffixes) + if(MSVC AND ${CMAKE_SIZEOF_VOID_P} EQUAL 8) + set(_path_suffixes PATH_SUFFIXES lib/x64) + else() + set(_path_suffixes PATH_SUFFIXES lib/Win32) + endif() # dynamic libs have different suffix in mac and linux if(APPLE) set(CUDNN_LIB_NAME "libcudnn.dylib") + elseif(MSVC) + set(CUDNN_LIB_NAME "cudnn") else() set(CUDNN_LIB_NAME "libcudnn.so") endif() @@ -186,6 +200,7 @@ function(detect_cuDNN) get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH) find_library(CUDNN_LIBRARY NAMES ${CUDNN_LIB_NAME} PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT} ${CUDNN_INCLUDE} ${__libpath_hist} ${__libpath_hist}/../lib + ${_path_suffixes} DOC "Path to cuDNN library.") if(CUDNN_INCLUDE AND CUDNN_LIBRARY) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index c48255c89f2..3dafb8d3514 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -7,8 +7,13 @@ set(Caffe_COMPILE_OPTIONS "") # ---[ Boost find_package(Boost 1.54 REQUIRED COMPONENTS system thread filesystem) list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${Boost_INCLUDE_DIRS}) +list(APPEND Caffe_DEFINITIONS PUBLIC -DBOOST_ALL_NO_LIB) list(APPEND Caffe_LINKER_LIBS PUBLIC ${Boost_LIBRARIES}) +if(DEFINED MSVC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18.0.40629.0) + # Required for VS 2013 Update 4 or earlier. + list(APPEND Caffe_DEFINITIONS PUBLIC -DBOOST_NO_CXX11_TEMPLATE_ALIASES) +endif() # ---[ Threads find_package(Threads REQUIRED) list(APPEND Caffe_LINKER_LIBS PRIVATE ${CMAKE_THREAD_LIBS_INIT}) @@ -43,7 +48,17 @@ list(APPEND Caffe_LINKER_LIBS PUBLIC ${GFLAGS_LIBRARIES}) include(cmake/ProtoBuf.cmake) # ---[ HDF5 -find_package(HDF5 COMPONENTS HL REQUIRED) +if(MSVC) + # Find HDF5 using it's hdf5-config.cmake file with MSVC + if(DEFINED HDF5_DIR) + list(APPEND CMAKE_MODULE_PATH ${HDF5_DIR}) + endif() + find_package(HDF5 COMPONENTS C HL REQUIRED) + set(HDF5_LIBRARIES hdf5-shared) + set(HDF5_HL_LIBRARIES hdf5_hl-shared) +else() + find_package(HDF5 COMPONENTS HL REQUIRED) +endif() list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${HDF5_INCLUDE_DIRS}) list(APPEND Caffe_LINKER_LIBS PUBLIC ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) @@ -86,7 +101,7 @@ if(NOT HAVE_CUDA) endif() if(USE_NCCL) - find_package(NCCL REQUIRED) + include("cmake/External/nccl.cmake") include_directories(SYSTEM ${NCCL_INCLUDE_DIR}) list(APPEND Caffe_LINKER_LIBS ${NCCL_LIBRARIES}) add_definitions(-DUSE_NCCL) @@ -173,6 +188,9 @@ if(BUILD_python) endif() if(PYTHONLIBS_FOUND AND NUMPY_FOUND AND Boost_PYTHON_FOUND) set(HAVE_PYTHON TRUE) + if(Boost_USE_STATIC_LIBS AND MSVC) + list(APPEND Caffe_DEFINITIONS PUBLIC -DBOOST_PYTHON_STATIC_LIB) + endif() if(BUILD_python_layer) list(APPEND Caffe_DEFINITIONS PRIVATE -DWITH_PYTHON_LAYER) list(APPEND Caffe_INCLUDE_DIRS PRIVATE ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} PUBLIC ${Boost_INCLUDE_DIRS}) @@ -183,11 +201,17 @@ endif() # ---[ Matlab if(BUILD_matlab) - find_package(MatlabMex) - if(MATLABMEX_FOUND) - set(HAVE_MATLAB TRUE) + if(MSVC) + find_package(Matlab COMPONENTS MAIN_PROGRAM MX_LIBRARY) + if(MATLAB_FOUND) + set(HAVE_MATLAB TRUE) + endif() + else() + find_package(MatlabMex) + if(MATLABMEX_FOUND) + set(HAVE_MATLAB TRUE) + endif() endif() - # sudo apt-get install liboctave-dev find_program(Octave_compiler NAMES mkoctfile DOC "Octave C++ compiler") diff --git a/cmake/External/nccl.cmake b/cmake/External/nccl.cmake new file mode 100644 index 00000000000..5f97f292239 --- /dev/null +++ b/cmake/External/nccl.cmake @@ -0,0 +1,35 @@ +# if (NOT __NCCL_INCLUDED) # guard against multiple includes + set(__NCCL_INCLUDED TRUE) + if(MSVC) + # use the system-wide nccl if present + find_package(NCCL) + if (NCCL_FOUND) + set(NCCL_EXTERNAL FALSE) + else() + # build directory + set(nccl_PREFIX ${CMAKE_BINARY_DIR}/external/nccl-prefix) + # install directory + set(nccl_INSTALL ${CMAKE_BINARY_DIR}/external/nccl-install) + ExternalProject_Add(nccl + PREFIX ${nccl_PREFIX} + URL https://github.com/willyd/nccl/archive/470b3130457f125f4608c7baee71123aa16a3b12.zip + UPDATE_COMMAND "" + INSTALL_DIR ${nccl_INSTALL} + CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${nccl_INSTALL} + -DBUILD_SHARED_LIBS=OFF + -DNCCL_BUILD_TESTS:BOOL=OFF + + LOG_DOWNLOAD 1 + LOG_INSTALL 1 + BUILD_BYPRODUCTS ${nccl_INSTALL}/include ${nccl_INSTALL}/lib/nccl.lib + ) + + set(NCCL_INCLUDE_DIR ${nccl_INSTALL}/include) + set(NCCL_LIBRARIES ${nccl_INSTALL}/lib/nccl.lib) + endif() + else() + # default to find package on UNIX systems + find_package(NCCL REQUIRED) + endif() +# endif() \ No newline at end of file diff --git a/cmake/Modules/FindGFlags.cmake b/cmake/Modules/FindGFlags.cmake index 29b60f05037..f44ecc051f9 100644 --- a/cmake/Modules/FindGFlags.cmake +++ b/cmake/Modules/FindGFlags.cmake @@ -14,26 +14,15 @@ include(FindPackageHandleStandardArgs) set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags") # We are testing only a couple of files in the include directories -if(WIN32) - find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h - PATHS ${GFLAGS_ROOT_DIR}/src/windows) -else() - find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h - PATHS ${GFLAGS_ROOT_DIR}) -endif() +find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h + PATHS ${GFLAGS_ROOT_DIR}) if(MSVC) - find_library(GFLAGS_LIBRARY_RELEASE - NAMES libgflags - PATHS ${GFLAGS_ROOT_DIR} - PATH_SUFFIXES Release) - - find_library(GFLAGS_LIBRARY_DEBUG - NAMES libgflags-debug - PATHS ${GFLAGS_ROOT_DIR} - PATH_SUFFIXES Debug) + # rely on gflags-config.cmake + find_package(gflags NO_MODULE) - set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG}) + set(GFLAGS_LIBRARY ${gflags_LIBRARIES}) + set(GFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIRS}) else() find_library(GFLAGS_LIBRARY gflags) endif() diff --git a/cmake/Modules/FindGlog.cmake b/cmake/Modules/FindGlog.cmake index 99abbe478a0..eec263a795d 100644 --- a/cmake/Modules/FindGlog.cmake +++ b/cmake/Modules/FindGlog.cmake @@ -13,24 +13,15 @@ include(FindPackageHandleStandardArgs) set(GLOG_ROOT_DIR "" CACHE PATH "Folder contains Google glog") -if(WIN32) - find_path(GLOG_INCLUDE_DIR glog/logging.h - PATHS ${GLOG_ROOT_DIR}/src/windows) -else() - find_path(GLOG_INCLUDE_DIR glog/logging.h - PATHS ${GLOG_ROOT_DIR}) -endif() +find_path(GLOG_INCLUDE_DIR glog/logging.h + PATHS ${GLOG_ROOT_DIR}) if(MSVC) - find_library(GLOG_LIBRARY_RELEASE libglog_static - PATHS ${GLOG_ROOT_DIR} - PATH_SUFFIXES Release) - - find_library(GLOG_LIBRARY_DEBUG libglog_static - PATHS ${GLOG_ROOT_DIR} - PATH_SUFFIXES Debug) + # rely on glog-config.cmake + find_package(glog NO_MODULE) - set(GLOG_LIBRARY optimized ${GLOG_LIBRARY_RELEASE} debug ${GLOG_LIBRARY_DEBUG}) + set(GLOG_LIBRARY ${glog_LIBRARIES}) + set(GLOG_INCLUDE_DIR ${glog_INCLUDE_DIRS}) else() find_library(GLOG_LIBRARY glog PATHS ${GLOG_ROOT_DIR} diff --git a/cmake/Modules/FindLMDB.cmake b/cmake/Modules/FindLMDB.cmake index 8a817fd6f10..2f0adb1b6d6 100644 --- a/cmake/Modules/FindLMDB.cmake +++ b/cmake/Modules/FindLMDB.cmake @@ -12,8 +12,12 @@ # Copyright 2013 Conrad Steenberg # Aug 31, 2013 -find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS "$ENV{LMDB_DIR}/include") -find_library(LMDB_LIBRARIES NAMES lmdb PATHS "$ENV{LMDB_DIR}/lib" ) +if(MSVC) + find_package(LMDB NO_MODULE) +else() + find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS "$ENV{LMDB_DIR}/include") + find_library(LMDB_LIBRARIES NAMES lmdb PATHS "$ENV{LMDB_DIR}/lib" ) +endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LMDB DEFAULT_MSG LMDB_INCLUDE_DIR LMDB_LIBRARIES) diff --git a/cmake/Modules/FindLevelDB.cmake b/cmake/Modules/FindLevelDB.cmake index 97f08ac9349..6e6a92dd835 100644 --- a/cmake/Modules/FindLevelDB.cmake +++ b/cmake/Modules/FindLevelDB.cmake @@ -5,14 +5,20 @@ # LevelDB_FOUND - True if LevelDB found. # Look for the header file. -find_path(LevelDB_INCLUDE NAMES leveldb/db.h - PATHS $ENV{LEVELDB_ROOT}/include /opt/local/include /usr/local/include /usr/include - DOC "Path in which the file leveldb/db.h is located." ) - -# Look for the library. -find_library(LevelDB_LIBRARY NAMES leveldb - PATHS /usr/lib $ENV{LEVELDB_ROOT}/lib - DOC "Path to leveldb library." ) +if(MSVC) + find_package(LevelDB NO_MODULE) + set(LevelDB_INCLUDE ${LevelDB_INCLUDE_DIRS}) + set(LevelDB_LIBRARY ${LevelDB_LIBRARIES}) +else() + find_path(LevelDB_INCLUDE NAMES leveldb/db.h + PATHS $ENV{LEVELDB_ROOT}/include /opt/local/include /usr/local/include /usr/include + DOC "Path in which the file leveldb/db.h is located." ) + + # Look for the library. + find_library(LevelDB_LIBRARY NAMES leveldb + PATHS /usr/lib $ENV{LEVELDB_ROOT}/lib + DOC "Path to leveldb library." ) +endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LevelDB DEFAULT_MSG LevelDB_INCLUDE LevelDB_LIBRARY) diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake index 5ab93b2d6b6..8ac6fc0c1e3 100644 --- a/cmake/Modules/FindMKL.cmake +++ b/cmake/Modules/FindMKL.cmake @@ -1,110 +1,110 @@ -# Find the MKL libraries -# -# Options: -# -# MKL_USE_SINGLE_DYNAMIC_LIBRARY : use single dynamic library interface -# MKL_USE_STATIC_LIBS : use static libraries -# MKL_MULTI_THREADED : use multi-threading -# -# This module defines the following variables: -# -# MKL_FOUND : True mkl is found -# MKL_INCLUDE_DIR : unclude directory -# MKL_LIBRARIES : the libraries to link against. - - -# ---[ Options -caffe_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON) -caffe_option(MKL_USE_STATIC_LIBS "Use static libraries" OFF IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) -caffe_option(MKL_MULTI_THREADED "Use multi-threading" ON IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) - -# ---[ Root folders -set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder contains intel libs") -find_path(MKL_ROOT include/mkl.h PATHS $ENV{MKLROOT} ${INTEL_ROOT}/mkl - DOC "Folder contains MKL") - -# ---[ Find include dir -find_path(MKL_INCLUDE_DIR mkl.h PATHS ${MKL_ROOT} PATH_SUFFIXES include) -set(__looked_for MKL_INCLUDE_DIR) - -# ---[ Find libraries -if(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(__path_suffixes lib lib/ia32) -else() - set(__path_suffixes lib lib/intel64) -endif() - -set(__mkl_libs "") -if(MKL_USE_SINGLE_DYNAMIC_LIBRARY) - list(APPEND __mkl_libs rt) -else() - if(CMAKE_SIZEOF_VOID_P EQUAL 4) - if(WIN32) - list(APPEND __mkl_libs intel_c) - else() - list(APPEND __mkl_libs intel gf) - endif() - else() - list(APPEND __mkl_libs intel_lp64 gf_lp64) - endif() - - if(MKL_MULTI_THREADED) - list(APPEND __mkl_libs intel_thread) - else() - list(APPEND __mkl_libs sequential) - endif() - - list(APPEND __mkl_libs core cdft_core) -endif() - - -foreach (__lib ${__mkl_libs}) - set(__mkl_lib "mkl_${__lib}") - string(TOUPPER ${__mkl_lib} __mkl_lib_upper) - - if(MKL_USE_STATIC_LIBS) - set(__mkl_lib "lib${__mkl_lib}.a") - endif() - - find_library(${__mkl_lib_upper}_LIBRARY - NAMES ${__mkl_lib} - PATHS ${MKL_ROOT} "${MKL_INCLUDE_DIR}/.." - PATH_SUFFIXES ${__path_suffixes} - DOC "The path to Intel(R) MKL ${__mkl_lib} library") - mark_as_advanced(${__mkl_lib_upper}_LIBRARY) - - list(APPEND __looked_for ${__mkl_lib_upper}_LIBRARY) - list(APPEND MKL_LIBRARIES ${${__mkl_lib_upper}_LIBRARY}) -endforeach() - - -if(NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) - if (MKL_USE_STATIC_LIBS) - set(__iomp5_libs iomp5 libiomp5mt.lib) - else() - set(__iomp5_libs iomp5 libiomp5md.lib) - endif() - - if(WIN32) - find_path(INTEL_INCLUDE_DIR omp.h PATHS ${INTEL_ROOT} PATH_SUFFIXES include) - list(APPEND __looked_for INTEL_INCLUDE_DIR) - endif() - - find_library(MKL_RTL_LIBRARY ${__iomp5_libs} - PATHS ${INTEL_RTL_ROOT} ${INTEL_ROOT}/compiler ${MKL_ROOT}/.. ${MKL_ROOT}/../compiler - PATH_SUFFIXES ${__path_suffixes} - DOC "Path to Path to OpenMP runtime library") - - list(APPEND __looked_for MKL_RTL_LIBRARY) - list(APPEND MKL_LIBRARIES ${MKL_RTL_LIBRARY}) -endif() - - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(MKL DEFAULT_MSG ${__looked_for}) - -if(MKL_FOUND) - message(STATUS "Found MKL (include: ${MKL_INCLUDE_DIR}, lib: ${MKL_LIBRARIES}") -endif() - -caffe_clear_vars(__looked_for __mkl_libs __path_suffixes __lib_suffix __iomp5_libs) +# Find the MKL libraries +# +# Options: +# +# MKL_USE_SINGLE_DYNAMIC_LIBRARY : use single dynamic library interface +# MKL_USE_STATIC_LIBS : use static libraries +# MKL_MULTI_THREADED : use multi-threading +# +# This module defines the following variables: +# +# MKL_FOUND : True mkl is found +# MKL_INCLUDE_DIR : unclude directory +# MKL_LIBRARIES : the libraries to link against. + + +# ---[ Options +caffe_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON) +caffe_option(MKL_USE_STATIC_LIBS "Use static libraries" OFF IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) +caffe_option(MKL_MULTI_THREADED "Use multi-threading" ON IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) + +# ---[ Root folders +set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder contains intel libs") +find_path(MKL_ROOT include/mkl.h PATHS $ENV{MKLROOT} ${INTEL_ROOT}/mkl + DOC "Folder contains MKL") + +# ---[ Find include dir +find_path(MKL_INCLUDE_DIR mkl.h PATHS ${MKL_ROOT} PATH_SUFFIXES include) +set(__looked_for MKL_INCLUDE_DIR) + +# ---[ Find libraries +if(CMAKE_SIZEOF_VOID_P EQUAL 4) + set(__path_suffixes lib lib/ia32) +else() + set(__path_suffixes lib lib/intel64) +endif() + +set(__mkl_libs "") +if(MKL_USE_SINGLE_DYNAMIC_LIBRARY) + list(APPEND __mkl_libs rt) +else() + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + if(WIN32) + list(APPEND __mkl_libs intel_c) + else() + list(APPEND __mkl_libs intel gf) + endif() + else() + list(APPEND __mkl_libs intel_lp64 gf_lp64) + endif() + + if(MKL_MULTI_THREADED) + list(APPEND __mkl_libs intel_thread) + else() + list(APPEND __mkl_libs sequential) + endif() + + list(APPEND __mkl_libs core cdft_core) +endif() + + +foreach (__lib ${__mkl_libs}) + set(__mkl_lib "mkl_${__lib}") + string(TOUPPER ${__mkl_lib} __mkl_lib_upper) + + if(MKL_USE_STATIC_LIBS) + set(__mkl_lib "lib${__mkl_lib}.a") + endif() + + find_library(${__mkl_lib_upper}_LIBRARY + NAMES ${__mkl_lib} + PATHS ${MKL_ROOT} "${MKL_INCLUDE_DIR}/.." + PATH_SUFFIXES ${__path_suffixes} + DOC "The path to Intel(R) MKL ${__mkl_lib} library") + mark_as_advanced(${__mkl_lib_upper}_LIBRARY) + + list(APPEND __looked_for ${__mkl_lib_upper}_LIBRARY) + list(APPEND MKL_LIBRARIES ${${__mkl_lib_upper}_LIBRARY}) +endforeach() + + +if(NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) + if (MKL_USE_STATIC_LIBS) + set(__iomp5_libs iomp5 libiomp5mt.lib) + else() + set(__iomp5_libs iomp5 libiomp5md.lib) + endif() + + if(WIN32) + find_path(INTEL_INCLUDE_DIR omp.h PATHS ${INTEL_ROOT} PATH_SUFFIXES include) + list(APPEND __looked_for INTEL_INCLUDE_DIR) + endif() + + find_library(MKL_RTL_LIBRARY ${__iomp5_libs} + PATHS ${INTEL_RTL_ROOT} ${INTEL_ROOT}/compiler ${MKL_ROOT}/.. ${MKL_ROOT}/../compiler + PATH_SUFFIXES ${__path_suffixes} + DOC "Path to Path to OpenMP runtime library") + + list(APPEND __looked_for MKL_RTL_LIBRARY) + list(APPEND MKL_LIBRARIES ${MKL_RTL_LIBRARY}) +endif() + + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MKL DEFAULT_MSG ${__looked_for}) + +if(MKL_FOUND) + message(STATUS "Found MKL (include: ${MKL_INCLUDE_DIR}, lib: ${MKL_LIBRARIES}") +endif() + +caffe_clear_vars(__looked_for __mkl_libs __path_suffixes __lib_suffix __iomp5_libs) diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake index a6512ae7e4e..58e9aee8fbf 100644 --- a/cmake/Modules/FindOpenBLAS.cmake +++ b/cmake/Modules/FindOpenBLAS.cmake @@ -28,8 +28,13 @@ SET(Open_BLAS_LIB_SEARCH_PATHS $ENV{OpenBLAS_HOME}/lib ) +if(MSVC) + set(OpenBLAS_LIB_NAMES libopenblas.dll.a) +else() + set(OpenBLAS_LIB_NAMES openblas) +endif() FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS}) -FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) +FIND_LIBRARY(OpenBLAS_LIB NAMES ${OpenBLAS_LIB_NAMES} PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) SET(OpenBLAS_FOUND ON) diff --git a/cmake/Modules/FindSnappy.cmake b/cmake/Modules/FindSnappy.cmake index eff2a864a7b..3e4f5e6f636 100644 --- a/cmake/Modules/FindSnappy.cmake +++ b/cmake/Modules/FindSnappy.cmake @@ -7,12 +7,16 @@ # SNAPPY_FOUND # Snappy_INCLUDE_DIR # Snappy_LIBRARIES +if(MSVC) + # rely on snappy-config.cmake + find_package(Snappy NO_MODULE) +else() + find_path(Snappy_INCLUDE_DIR NAMES snappy.h + PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/include) -find_path(Snappy_INCLUDE_DIR NAMES snappy.h - PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/include) - -find_library(Snappy_LIBRARIES NAMES snappy - PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/lib) + find_library(Snappy_LIBRARIES NAMES snappy + PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/lib) +endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_INCLUDE_DIR Snappy_LIBRARIES) diff --git a/cmake/ProtoBuf.cmake b/cmake/ProtoBuf.cmake index 8005b448707..a0d429505b8 100644 --- a/cmake/ProtoBuf.cmake +++ b/cmake/ProtoBuf.cmake @@ -1,7 +1,13 @@ # Finds Google Protocol Buffers library and compilers and extends # the standard cmake script with version and python generation support -find_package( Protobuf REQUIRED ) +if(MSVC) + # search using protobuf-config.cmake + find_package( Protobuf REQUIRED NO_MODULE) + set(PROTOBUF_INCLUDE_DIR ${PROTOBUF_INCLUDE_DIRS}) +else() + find_package( Protobuf REQUIRED ) +endif() list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${PROTOBUF_INCLUDE_DIR}) list(APPEND Caffe_LINKER_LIBS PUBLIC ${PROTOBUF_LIBRARIES}) diff --git a/cmake/TargetResolvePrerequesites.cmake b/cmake/TargetResolvePrerequesites.cmake new file mode 100644 index 00000000000..429c113958a --- /dev/null +++ b/cmake/TargetResolvePrerequesites.cmake @@ -0,0 +1,189 @@ +set(THIS_FILE ${CMAKE_CURRENT_LIST_FILE}) +set(THIS_DIR ${CMAKE_CURRENT_LIST_DIR}) + +include(CMakeParseArguments) + +function(caffe_prerequisites_directories VAR) + if(BUILD_SHARED_LIBS) + # Append the caffe library output directory + list(APPEND _directories $) + endif() + # Add boost to search directories + list(APPEND _directories ${Boost_LIBRARY_DIRS}) + # Add gflags to search directories + # gflags_DIR should point to root/CMake + get_filename_component(_dir ${gflags_DIR} DIRECTORY) + list(APPEND _directories ${_dir}/lib) + # Add glog to search directories + # glog_DIR should point to root/lib/cmake/glog + get_filename_component(_dir ${glog_DIR} DIRECTORY) + get_filename_component(_dir ${_dir} DIRECTORY) + get_filename_component(_dir ${_dir} DIRECTORY) + list(APPEND _directories ${_dir}/bin) + # Add HDF5 to search directories + # HDF5_DIR should point to root/CMake + get_filename_component(_dir ${HDF5_DIR} DIRECTORY) + list(APPEND _directories ${_dir}/bin) + # Add OpenCV to search directories + get_filename_component(_dir ${OpenCV_LIB_PATH} DIRECTORY) + list(APPEND _directories ${_dir}/bin) + if(CUDNN_FOUND AND HAVE_CUDNN) + # Add OpenCV to search directories + get_filename_component(_dir ${CUDNN_LIBRARY} DIRECTORY) + get_filename_component(_dir ${_dir} DIRECTORY) + get_filename_component(_dir ${_dir} DIRECTORY) + list(APPEND _directories ${_dir}/bin) + endif() + if(USE_NCCL) + # add the nvml.dll path if we are using nccl + file(TO_CMAKE_PATH "$ENV{NVTOOLSEXT_PATH}" _nvtools_ext) + if(NOT "${_nvtools_ext}" STREQUAL "") + get_filename_component(_nvsmi_path ${_nvtools_ext}/../nvsmi ABSOLUTE) + list(APPEND _directories ${_nvsmi_path}) + endif() + endif() + list(REMOVE_DUPLICATES _directories) + set(${VAR} ${_directories} PARENT_SCOPE) +endfunction() + +function(caffe_copy_prerequisites target) + caffe_prerequisites_directories(_directories) + target_copy_prerequisites(${target} ${ARGN} DIRECTORIES ${_directories}) +endfunction() + +function(caffe_install_prerequisites target) + caffe_prerequisites_directories(_directories) + target_install_prerequisites(${target} ${ARGN} DIRECTORIES ${_directories}) +endfunction() + +function(target_copy_prerequisites target) + set(options USE_HARD_LINKS) + set(oneValueArgs DESTINATION) + set(multiValueArgs DIRECTORIES) + cmake_parse_arguments(tcp "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN}) + if(NOT tcp_DESTINATION) + set(tcp_DESTINATION $) + endif() + string(REPLACE ";" "@@" tcp_DIRECTORIES "${tcp_DIRECTORIES}") + if(USE_NCCL) + # nccl loads the nvml.dll dynamically so we need + # to list it explicitely + list(APPEND _plugins nvml.dll) + endif() + string(REPLACE ";" "@@" _plugins "${_plugins}") + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${CMAKE_COMMAND} + -DTARGET=$ + -DDESTINATION=${tcp_DESTINATION} + -DUSE_HARD_LINKS=${tcp_USE_HARD_LINKS} + -DDIRECTORIES=${tcp_DIRECTORIES} + -DPLUGINS=${_plugins} + -P ${THIS_FILE} + ) +endfunction() + +function(target_install_prerequisites target) + set(options ) + set(oneValueArgs DESTINATION) + set(multiValueArgs DIRECTORIES) + cmake_parse_arguments(tcp "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN}) + if(NOT tcp_DESTINATION) + set(tcp_DESTINATION bin) + endif() + if(NOT IS_ABSOLUTE ${tcp_DESTINATION}) + set(tcp_DESTINATION ${CMAKE_INSTALL_PREFIX}/${tcp_DESTINATION}) + endif() + string(REPLACE ";" "@@" tcp_DIRECTORIES "${tcp_DIRECTORIES}") + if(USE_NCCL) + # nccl loads the nvml.dll dynamically so we need + # to list it explicitely + list(APPEND _plugins nvml.dll) + endif() + string(REPLACE ";" "@@" _plugins "${_plugins}") + set(_command_output ${CMAKE_CURRENT_BINARY_DIR}/${target}-install-prerequisites.stamp) + add_custom_command(OUTPUT ${_command_output} + COMMAND ${CMAKE_COMMAND} + -DTARGET=$ + -DDESTINATION=${tcp_DESTINATION} + -DUSE_HARD_LINKS=0 + -DDIRECTORIES=${tcp_DIRECTORIES} + -DPLUGINS=${_plugins} + -P ${THIS_FILE} + COMMAND ${CMAKE_COMMAND} -E touch ${_command_output} + ) + add_custom_target(${target}_install_prerequisites ALL + DEPENDS ${_command_output}) + install(FILES ${_command_output} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tmp) +endfunction() + +function(create_hardlink link target result_variable) + file(TO_NATIVE_PATH ${link} _link) + file(TO_NATIVE_PATH ${target} _target) + execute_process(COMMAND cmd /c mklink /H "${_link}" "${_target}" + RESULT_VARIABLE _result + OUTPUT_VARIABLE _stdout + ERROR_VARIABLE _stderr + ) + set(${result_variable} ${_result} PARENT_SCOPE) +endfunction() + +function(copy_changed_file filename destination use_hard_links) + set(_copy 1) + set(_src_name ${filename}) + get_filename_component(_name ${_src_name} NAME) + set(_dst_name ${destination}/${_name}) + + # lock a file to ensure that no two cmake processes + # try to copy the same file at the same time in parallel + # builds + string(SHA1 _hash ${_dst_name}) + set(_lock_file ${CMAKE_BINARY_DIR}/${_hash}.lock) + file(LOCK ${_lock_file} GUARD FUNCTION) + + if(EXISTS ${_dst_name}) + file(TIMESTAMP ${_dst_name} _dst_time) + file(TIMESTAMP ${_src_name} _src_time) + if(${_dst_time} STREQUAL ${_src_time}) + # skip this library if the destination and source + # have the same time stamp + return() + else() + # file has changed remove + file(REMOVE ${_dst_name}) + endif() + endif() + + if(use_hard_links) + message(STATUS "Creating hardlink for ${_name} in ${destination}") + create_hardlink(${_dst_name} ${_src_name} _result) + if(_result EQUAL 0) + set(_copy 0) + else() + message(STATUS "Failed to create hardlink ${_dst_name}. Copying instead.") + endif() + endif() + if(_copy) + message(STATUS "Copying ${_name} to ${destination}") + file(COPY ${_src_name} DESTINATION ${DESTINATION}) + endif() +endfunction() + + +if(CMAKE_SCRIPT_MODE_FILE) + include(${THIS_DIR}/CaffeGetPrerequisites.cmake) + # Recreate a list by replacing the @@ with ; + string(REPLACE "@@" ";" DIRECTORIES "${DIRECTORIES}") + string(REPLACE "@@" ";" PLUGINS "${PLUGINS}") + # Get a recursive list of dependencies required by target using dumpbin + get_prerequisites(${TARGET} _prerequisites 1 1 "" "${DIRECTORIES}") + foreach(_prereq ${_prerequisites} ${PLUGINS}) + # Resolve the dependency using the list of directories + gp_resolve_item("${TARGET}" "${_prereq}" "" "${DIRECTORIES}" resolved_file) + # Copy or create hardlink (if possible) + if(EXISTS ${resolved_file}) + copy_changed_file(${resolved_file} ${DESTINATION} ${USE_HARD_LINKS}) + endif() + endforeach() +endif() \ No newline at end of file diff --git a/cmake/Targets.cmake b/cmake/Targets.cmake index 090f86c5500..3755461113e 100644 --- a/cmake/Targets.cmake +++ b/cmake/Targets.cmake @@ -1,7 +1,16 @@ ################################################################################################ # Defines global Caffe_LINK flag, This flag is required to prevent linker from excluding # some objects which are not addressed directly but are registered via static constructors -macro(caffe_set_caffe_link) +macro(caffe_set_caffe_link) + if(MSVC AND CMAKE_GENERATOR MATCHES Ninja) + foreach(_suffix "" ${CMAKE_CONFIGURATION_TYPES}) + if(NOT _suffix STREQUAL "") + string(TOUPPER _${_suffix} _suffix) + endif() + set(CMAKE_CXX_FLAGS${_suffix} "${CMAKE_CXX_FLAGS${_suffix}} /FS") + set(CMAKE_C_FLAGS${_suffix} "${CMAKE_C_FLAGS${_suffix}} /FS") + endforeach() + endif() if(BUILD_SHARED_LIBS) set(Caffe_LINK caffe) else() @@ -9,6 +18,8 @@ macro(caffe_set_caffe_link) set(Caffe_LINK -Wl,-force_load caffe) elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") set(Caffe_LINK -Wl,--whole-archive caffe -Wl,--no-whole-archive) + elseif(MSVC) + set(Caffe_LINK caffe) endif() endif() endmacro() @@ -52,9 +63,18 @@ endfunction() # caffe_pickup_caffe_sources() function(caffe_pickup_caffe_sources root) # put all files in source groups (visible as subfolder in many IDEs) + set(caffe_export_hdr_in ${PROJECT_SOURCE_DIR}/cmake/Templates/export.hpp.in) + set(caffe_export_hdr ${PROJECT_BINARY_DIR}/caffe/export.hpp) + set(caffe_symbols_hdr ${PROJECT_BINARY_DIR}/caffe/include_symbols.hpp) + set_source_files_properties(${caffe_export_hdr} ${caffe_symbols_hdr} PROPERTIES GENERATED TRUE) + caffe_source_group("Include" GLOB "${root}/include/caffe/*.h*") caffe_source_group("Include\\Util" GLOB "${root}/include/caffe/util/*.h*") caffe_source_group("Include" GLOB "${PROJECT_BINARY_DIR}/caffe_config.h*") + caffe_source_group("Include" GLOB "${caffe_export_hdr}") + if(MSVC AND NOT BUILD_SHARED_LIBS) + caffe_source_group("Include" GLOB "${caffe_symbols_hdr}") + endif() caffe_source_group("Source" GLOB "${root}/src/caffe/*.cpp") caffe_source_group("Source\\Util" GLOB "${root}/src/caffe/util/*.cpp") caffe_source_group("Source\\Layers" GLOB "${root}/src/caffe/layers/*.cpp") @@ -76,7 +96,13 @@ function(caffe_pickup_caffe_sources root) list(REMOVE_ITEM srcs ${test_srcs}) # adding headers to make the visible in some IDEs (Qt, VS, Xcode) - list(APPEND srcs ${hdrs} ${PROJECT_BINARY_DIR}/caffe_config.h) + list(APPEND srcs ${hdrs} + ${PROJECT_BINARY_DIR}/caffe_config.h + ${caffe_export_hdr} + ) + if(MSVC AND NOT BUILD_SHARED_LIBS) + list(APPEND srcs ${caffe_symbols_hdr}) + endif() list(APPEND test_srcs ${test_hdrs}) # collect cuda files @@ -99,6 +125,10 @@ function(caffe_pickup_caffe_sources root) set(cuda ${cuda} PARENT_SCOPE) set(test_srcs ${test_srcs} PARENT_SCOPE) set(test_cuda ${test_cuda} PARENT_SCOPE) + set(caffe_export_hdr_in ${caffe_export_hdr_in} PARENT_SCOPE) + set(caffe_export_hdr ${caffe_export_hdr} PARENT_SCOPE) + set(caffe_symbols_hdr ${caffe_symbols_hdr} PARENT_SCOPE) + endfunction() ################################################################################################ diff --git a/cmake/Templates/CaffeConfig.cmake.in b/cmake/Templates/CaffeConfig.cmake.in index 77c4059e560..04d5d978e38 100644 --- a/cmake/Templates/CaffeConfig.cmake.in +++ b/cmake/Templates/CaffeConfig.cmake.in @@ -37,6 +37,30 @@ if(@USE_OPENCV@) endif() endif() +# Handle other imported targets libraries +if(@GFLAGS_IMPORTED@) + find_package(gflags REQUIRED NO_MODULE) +endif() + +if(@GLOG_IMPORTED@) + find_package(glog REQUIRED NO_MODULE) +endif() + +if(@HDF5_IMPORTED@) + find_package(HDF5 COMPONENTS C HL REQUIRED NO_MODULE) +endif() + +if(@USE_LMDB@ AND @LMDB_IMPORTED@) + find_package(LMDB REQUIRED NO_MODULE) +endif() + +if(@USE_LEVELDB@ AND @LEVELDB_IMPORTED@) + find_package(LevelDB REQUIRED NO_MODULE) + if(@SNAPPY_IMPORTED@) + find_package(Snappy REQUIRED NO_MODULE) + endif() +endif() + # Compute paths get_filename_component(Caffe_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) diff --git a/cmake/Templates/export.hpp.in b/cmake/Templates/export.hpp.in new file mode 100644 index 00000000000..33ff222c156 --- /dev/null +++ b/cmake/Templates/export.hpp.in @@ -0,0 +1,10 @@ +#ifndef CAFFE_EXPORT_HPP_ +#define CAFFE_EXPORT_HPP_ + +// CAFFE_BUILDING_STATIC_LIB should be defined +// only by the caffe target +#if defined(_MSC_VER) && !defined(CAFFE_BUILDING_STATIC_LIB) + ${CAFFE_INCLUDE_SYMBOLS} +#endif + +#endif // CAFFE_EXPORT_HPP_ \ No newline at end of file diff --git a/cmake/WindowsCreateLinkHeader.cmake b/cmake/WindowsCreateLinkHeader.cmake new file mode 100644 index 00000000000..29e77b08953 --- /dev/null +++ b/cmake/WindowsCreateLinkHeader.cmake @@ -0,0 +1,72 @@ +set(_windows_create_link_header "${CMAKE_CURRENT_LIST_FILE}") + +# function to add a post build command to create a link header +function(windows_create_link_header target outputfile) + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${CMAKE_COMMAND} + #-DCMAKE_GENERATOR=${CMAKE_GENERATOR} + -DMSVC_VERSION=${MSVC_VERSION} + -DTARGET_FILE=$ + #-DPROJECT_BINARY_DIR=${PROJECT_BINARY_DIR} + #-DCMAKE_CURRENT_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR} + #-DCONFIGURATION=$ + -DOUTPUT_FILE=${outputfile} + -P ${_windows_create_link_header} + BYPRODUCTS ${outputfile} + ) +endfunction() + + +function(find_dumpbin var) + # MSVC_VERSION = + # 1200 = VS 6.0 + # 1300 = VS 7.0 + # 1310 = VS 7.1 + # 1400 = VS 8.0 + # 1500 = VS 9.0 + # 1600 = VS 10.0 + # 1700 = VS 11.0 + # 1800 = VS 12.0 + # 1900 = VS 14.0 + set(MSVC_PRODUCT_VERSION_1200 6.0) + set(MSVC_PRODUCT_VERSION_1300 7.0) + set(MSVC_PRODUCT_VERSION_1310 7.1) + set(MSVC_PRODUCT_VERSION_1400 8.0) + set(MSVC_PRODUCT_VERSION_1500 9.0) + set(MSVC_PRODUCT_VERSION_1600 10.0) + set(MSVC_PRODUCT_VERSION_1700 11.0) + set(MSVC_PRODUCT_VERSION_1800 12.0) + set(MSVC_PRODUCT_VERSION_1900 14.0) + get_filename_component(MSVC_VC_DIR [HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\${MSVC_PRODUCT_VERSION_${MSVC_VERSION}}\\Setup\\VC;ProductDir] REALPATH CACHE) + + find_program(DUMPBIN_EXECUTABLE dumpbin ${MSVC_VC_DIR}/bin) + if(NOT DUMPBIN_EXECUTABLE) + message(FATAL_ERROR "Could not find DUMPBIN_EXECUTABLE please define this variable") + endif() + set(${var} ${DUMPBIN_EXECUTABLE} PARENT_SCOPE) +endfunction() + +macro(print_date) + execute_process(COMMAND powershell -NoProfile -Command "get-date") +endmacro() + + +if(CMAKE_SCRIPT_MODE_FILE) + cmake_policy(SET CMP0007 NEW) + # find the dumpbin exe + find_dumpbin(dumpbin) + # execute dumpbin to generate a list of symbols + execute_process(COMMAND ${dumpbin} /SYMBOLS ${TARGET_FILE} + RESULT_VARIABLE _result + OUTPUT_VARIABLE _output + ERROR_VARIABLE _error + ) + # match all layers and solvers instantiation guard + string(REGEX MATCHALL "\\?gInstantiationGuard[^\\(\\) ]*" __symbols ${_output}) + # define a string to generate a list of pragmas + foreach(__symbol ${__symbols}) + set(__pragma "${__pragma}#pragma comment(linker, \"/include:${__symbol}\")\n") + endforeach() + file(WRITE ${OUTPUT_FILE} ${__pragma}) +endif() + diff --git a/cmake/WindowsDownloadPrebuiltDependencies.cmake b/cmake/WindowsDownloadPrebuiltDependencies.cmake new file mode 100644 index 00000000000..168cf319115 --- /dev/null +++ b/cmake/WindowsDownloadPrebuiltDependencies.cmake @@ -0,0 +1,92 @@ +set(DEPENDENCIES_VERSION 1.1.0) +set(DEPENDENCIES_NAME_1800_27 libraries_v120_x64_py27_${DEPENDENCIES_VERSION}) +set(DEPENDENCIES_NAME_1900_27 libraries_v140_x64_py27_${DEPENDENCIES_VERSION}) +set(DEPENDENCIES_NAME_1900_35 libraries_v140_x64_py35_${DEPENDENCIES_VERSION}) + +set(DEPENDENCIES_URL_BASE https://github.com/willyd/caffe-builder/releases/download) +set(DEPENDENCIES_FILE_EXT .tar.bz2) +set(DEPENDENCIES_URL_1800_27 "${DEPENDENCIES_URL_BASE}/v${DEPENDENCIES_VERSION}/${DEPENDENCIES_NAME_1800_27}${DEPENDENCIES_FILE_EXT}") +set(DEPENDENCIES_SHA_1800_27 "ba833d86d19b162a04d68b09b06df5e0dad947d4") +set(DEPENDENCIES_URL_1900_27 "${DEPENDENCIES_URL_BASE}/v${DEPENDENCIES_VERSION}/${DEPENDENCIES_NAME_1900_27}${DEPENDENCIES_FILE_EXT}") +set(DEPENDENCIES_SHA_1900_27 "17eecb095bd3b0774a87a38624a77ce35e497cd2") +set(DEPENDENCIES_URL_1900_35 "${DEPENDENCIES_URL_BASE}/v${DEPENDENCIES_VERSION}/${DEPENDENCIES_NAME_1900_35}${DEPENDENCIES_FILE_EXT}") +set(DEPENDENCIES_SHA_1900_35 "f060403fd1a7448d866d27c0e5b7dced39c0a607") + +caffe_option(USE_PREBUILT_DEPENDENCIES "Download and use the prebuilt dependencies" ON IF MSVC) +if(MSVC) + file(TO_CMAKE_PATH $ENV{USERPROFILE} USERPROFILE_DIR) + if(NOT EXISTS ${USERPROFILE_DIR}) + message(FATAL_ERROR "Could not find %USERPROFILE% directory. Please specify an alternate CAFFE_DEPENDENCIES_ROOT_DIR") + endif() + set(CAFFE_DEPENDENCIES_ROOT_DIR ${USERPROFILE_DIR}/.caffe/dependencies CACHE PATH "Prebuild depdendencies root directory") + set(CAFFE_DEPENDENCIES_DOWNLOAD_DIR ${CAFFE_DEPENDENCIES_ROOT_DIR}/download CACHE PATH "Download directory for prebuilt dependencies") +endif() +if(USE_PREBUILT_DEPENDENCIES) + # Determine the python version + if(BUILD_python) + if(NOT PYTHONINTERP_FOUND) + if(NOT "${python_version}" VERSION_LESS "3.0.0") + find_package(PythonInterp 3.5) + else() + find_package(PythonInterp 2.7) + endif() + endif() + set(_pyver ${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}) + else() + message(STATUS "Building without python. Prebuilt dependencies will default to Python 2.7") + set(_pyver 27) + endif() + if(NOT DEFINED DEPENDENCIES_URL_${MSVC_VERSION}_${_pyver}) + message(FATAL_ERROR "Could not find url for MSVC version = ${MSVC_VERSION} and Python version = ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}.") + endif() + # set the dependencies URL and SHA1 + set(DEPENDENCIES_URL ${DEPENDENCIES_URL_${MSVC_VERSION}_${_pyver}}) + set(DEPENDENCIES_SHA ${DEPENDENCIES_SHA_${MSVC_VERSION}_${_pyver}}) + set(CAFFE_DEPENDENCIES_DIR ${CAFFE_DEPENDENCIES_ROOT_DIR}/${DEPENDENCIES_NAME_${MSVC_VERSION}_${_pyver}}) + + foreach(_dir ${CAFFE_DEPENDENCIES_ROOT_DIR} + ${CAFFE_DEPENDENCIES_DOWNLOAD_DIR} + ${CAFFE_DEPENDENCIES_DIR}) + # create the directory if it does not exist + if(NOT EXISTS ${_dir}) + file(MAKE_DIRECTORY ${_dir}) + endif() + endforeach() + # download and extract the file if it does not exist or if does not match the sha1 + get_filename_component(_download_filename ${DEPENDENCIES_URL} NAME) + set(_download_path ${CAFFE_DEPENDENCIES_DOWNLOAD_DIR}/${_download_filename}) + set(_download_file 1) + if(EXISTS ${_download_path}) + file(SHA1 ${_download_path} _file_sha) + if("${_file_sha}" STREQUAL "${DEPENDENCIES_SHA}") + set(_download_file 0) + else() + set(_download_file 1) + message(STATUS "Removing file because sha1 does not match.") + file(REMOVE ${_download_path}) + endif() + endif() + if(_download_file) + message(STATUS "Downloading prebuilt dependencies to ${_download_path}") + file(DOWNLOAD "${DEPENDENCIES_URL}" + "${_download_path}" + EXPECTED_HASH SHA1=${DEPENDENCIES_SHA} + SHOW_PROGRESS + ) + if(EXISTS ${CAFFE_DEPENDENCIES_DIR}/libraries) + file(REMOVE_RECURSE ${CAFFE_DEPENDENCIES_DIR}/libraries) + endif() + endif() + if(EXISTS ${_download_path} AND NOT EXISTS ${CAFFE_DEPENDENCIES_DIR}/libraries) + message(STATUS "Extracting dependencies") + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xjf ${_download_path} + WORKING_DIRECTORY ${CAFFE_DEPENDENCIES_DIR} + ) + endif() + if(EXISTS ${CAFFE_DEPENDENCIES_DIR}/libraries/caffe-builder-config.cmake) + include(${CAFFE_DEPENDENCIES_DIR}/libraries/caffe-builder-config.cmake) + else() + message(FATAL_ERROR "Something went wrong while dowloading dependencies could not open caffe-builder-config.cmake") + endif() +endif() + diff --git a/cmake/lint.cmake b/cmake/lint.cmake index 70a006572bb..8cca27d5248 100644 --- a/cmake/lint.cmake +++ b/cmake/lint.cmake @@ -1,6 +1,10 @@ set(CMAKE_SOURCE_DIR ..) -set(LINT_COMMAND ${CMAKE_SOURCE_DIR}/scripts/cpp_lint.py) +set(python_executable) +if(WIN32) + set(python_executable ${PYTHON_EXECUTABLE}) +endif() +set(LINT_COMMAND ${python_executable} ${CMAKE_SOURCE_DIR}/scripts/cpp_lint.py) set(SRC_FILE_EXTENSIONS h hpp hu c cpp cu cc) set(EXCLUDE_FILE_EXTENSTIONS pb.h pb.cc) set(LINT_DIRS include src/caffe examples tools python matlab) @@ -22,7 +26,9 @@ foreach(ext ${EXCLUDE_FILE_EXTENSTIONS}) endforeach() # exclude generated pb files -list(REMOVE_ITEM LINT_SOURCES ${EXCLUDED_FILES}) +if(EXCLUDED_FILES) + list(REMOVE_ITEM LINT_SOURCES ${EXCLUDED_FILES}) +endif() execute_process( COMMAND ${LINT_COMMAND} ${LINT_SOURCES} diff --git a/data/mnist/get_mnist.ps1 b/data/mnist/get_mnist.ps1 new file mode 100644 index 00000000000..1fe464b039a --- /dev/null +++ b/data/mnist/get_mnist.ps1 @@ -0,0 +1,24 @@ +# This scripts downloads the mnist data and unzips it. +$ErrorActionPreference = 'Stop' + +pushd $PSScriptRoot + +echo "Downloading..." + +# get the path to 7-zip from the registry +$7zip = Join-Path (get-item HKLM:\SOFTWARE\7-Zip).GetValue('Path') '7z.exe' + +$fnames = @("train-images-idx3-ubyte", + "train-labels-idx1-ubyte", + "t10k-images-idx3-ubyte", + "t10k-labels-idx1-ubyte") + +foreach($fname in $fnames) { + if(-not (Test-Path $fname)) { + # Start-BitsTransfer -Source "http://yann.lecun.com/exdb/mnist/$fname.gz" -Destination "$fname.gz" + wget -Uri "http://yann.lecun.com/exdb/mnist/$fname.gz" -OutFile "$fname.gz" + . $7zip x "$fname.gz" + } +} + +popd \ No newline at end of file diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 43bbcb83789..fb45b3b703e 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -3,15 +3,19 @@ file(GLOB_RECURSE examples_srcs "${PROJECT_SOURCE_DIR}/examples/*.cpp") foreach(source_file ${examples_srcs}) # get file name get_filename_component(name ${source_file} NAME_WE) - + # get folder name get_filename_component(path ${source_file} PATH) get_filename_component(folder ${path} NAME_WE) - + add_executable(${name} ${source_file}) target_link_libraries(${name} ${Caffe_LINK}) caffe_default_properties(${name}) + if(MSVC AND COPY_PREREQUISITES) + caffe_copy_prerequisites(${name} USE_HARD_LINKS) + endif() + # set back RUNTIME_OUTPUT_DIRECTORY set_target_properties(${name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/examples/${folder}") @@ -22,6 +26,10 @@ foreach(source_file ${examples_srcs}) install(TARGETS ${name} DESTINATION ${CMAKE_INSTALL_BINDIR}) + if(MSVC AND INSTALL_PREREQUISITES) + caffe_install_prerequisites(${name} DESTINATION bin) + endif() + if(UNIX OR APPLE) # Funny command to make tutorials work # TODO: remove in future as soon as naming is standardized everywhere diff --git a/examples/cpp_classification/classification.cpp b/examples/cpp_classification/classification.cpp index 6b67c537a47..8affe524a89 100644 --- a/examples/cpp_classification/classification.cpp +++ b/examples/cpp_classification/classification.cpp @@ -92,7 +92,7 @@ static bool PairCompare(const std::pair& lhs, static std::vector Argmax(const std::vector& v, int N) { std::vector > pairs; for (size_t i = 0; i < v.size(); ++i) - pairs.push_back(std::make_pair(v[i], i)); + pairs.push_back(std::make_pair(v[i], static_cast(i))); std::partial_sort(pairs.begin(), pairs.begin() + N, pairs.end(), PairCompare); std::vector result; diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 57ddef77074..fc66a51c6f2 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -16,6 +16,11 @@ #include #endif +#if defined(_MSC_VER) +#include +#define mkdir(X, Y) _mkdir(X) +#endif + #include #include diff --git a/examples/mnist/create_mnist.ps1 b/examples/mnist/create_mnist.ps1 new file mode 100644 index 00000000000..6bb4b2812ac --- /dev/null +++ b/examples/mnist/create_mnist.ps1 @@ -0,0 +1,33 @@ +# This script converts the mnist data into lmdb/leveldb format, +# depending on the value assigned to $BACKEND. +param( + [string]$BuildDir +) + +$ErrorActionPreference = 'Stop' + +$CaffeRoot = (Resolve-Path (Join-Path $PSScriptRoot ..\..)) +$EXAMPLE = "$CaffeRoot\examples\mnist" +$DATA = "$CaffeRoot\data\mnist" +if("$BuildDir" -eq "") { + $BuildDir = "$CaffeRoot\build" +} +$BUILD = "$BuildDir\examples\mnist" + +$BACKEND = "lmdb" + +echo "Creating $BACKEND..." + +if(Test-Path $EXAMPLE\mnist_train_$BACKEND) { + rm -Recurse -Force $EXAMPLE\mnist_train_$BACKEND +} +if(Test-Path $EXAMPLE\mnist_train_$BACKEND) { + rm -Recurse -Force $EXAMPLE\mnist_test_$BACKEND +} + +. $BUILD\convert_mnist_data.exe $DATA\train-images.idx3-ubyte ` + $DATA\train-labels.idx1-ubyte $EXAMPLE\mnist_train_$BACKEND --backend=$BACKEND +. $BUILD\convert_mnist_data.exe $DATA\t10k-images.idx3-ubyte ` + $DATA\t10k-labels.idx1-ubyte $EXAMPLE\mnist_test_$BACKEND --backend=$BACKEND + +echo "Done." diff --git a/examples/mnist/readme.md b/examples/mnist/readme.md index 35952155a30..11dbdf77c84 100644 --- a/examples/mnist/readme.md +++ b/examples/mnist/readme.md @@ -8,7 +8,7 @@ priority: 1 # Training LeNet on MNIST with Caffe -We will assume that you have Caffe successfully compiled. If not, please refer to the [Installation page](/installation.html). In this tutorial, we will assume that your Caffe installation is located at `CAFFE_ROOT`. +We will assume that you have Caffe successfully compiled. If not, please refer to the [Installation page](/installation.html). In this tutorial, we will assume that your Caffe installation is located at `CAFFE_ROOT`. On Windows use the powershell (`.ps1`) instead of the bash (`.sh`) scripts. ## Prepare Datasets diff --git a/examples/mnist/train_lenet.ps1 b/examples/mnist/train_lenet.ps1 new file mode 100644 index 00000000000..912c62cea15 --- /dev/null +++ b/examples/mnist/train_lenet.ps1 @@ -0,0 +1,10 @@ +param( + [string]$BuildDir +) + +$CaffeRoot = (Resolve-Path (Join-Path $PSScriptRoot ..\..)) +if("$BuildDir" -eq "") { + $BuildDir = "$CaffeRoot\build" +} + +. $BuildDir\tools\caffe.exe train --solver=examples\mnist\lenet_solver.prototxt $args diff --git a/include/caffe/common.hpp b/include/caffe/common.hpp index 4904d1d8661..68858e36a81 100644 --- a/include/caffe/common.hpp +++ b/include/caffe/common.hpp @@ -16,6 +16,9 @@ #include // pair #include +#ifdef CMAKE_WINDOWS_BUILD + #include "caffe/export.hpp" +#endif #include "caffe/util/device_alternate.hpp" // Convert macro to string diff --git a/include/caffe/layer_factory.hpp b/include/caffe/layer_factory.hpp index 2369c132911..8fd908a5631 100644 --- a/include/caffe/layer_factory.hpp +++ b/include/caffe/layer_factory.hpp @@ -58,72 +58,31 @@ class LayerRegistry { typedef shared_ptr > (*Creator)(const LayerParameter&); typedef std::map CreatorRegistry; - static CreatorRegistry& Registry() { - static CreatorRegistry* g_registry_ = new CreatorRegistry(); - return *g_registry_; - } + static CreatorRegistry& Registry(); // Adds a creator. - static void AddCreator(const string& type, Creator creator) { - CreatorRegistry& registry = Registry(); - CHECK_EQ(registry.count(type), 0) - << "Layer type " << type << " already registered."; - registry[type] = creator; - } + static void AddCreator(const string& type, Creator creator); // Get a layer using a LayerParameter. - static shared_ptr > CreateLayer(const LayerParameter& param) { - if (Caffe::root_solver()) { - LOG(INFO) << "Creating layer " << param.name(); - } - const string& type = param.type(); - CreatorRegistry& registry = Registry(); - CHECK_EQ(registry.count(type), 1) << "Unknown layer type: " << type - << " (known types: " << LayerTypeListString() << ")"; - return registry[type](param); - } - - static vector LayerTypeList() { - CreatorRegistry& registry = Registry(); - vector layer_types; - for (typename CreatorRegistry::iterator iter = registry.begin(); - iter != registry.end(); ++iter) { - layer_types.push_back(iter->first); - } - return layer_types; - } + static shared_ptr > CreateLayer(const LayerParameter& param); + + static vector LayerTypeList(); private: // Layer registry should never be instantiated - everything is done with its // static variables. - LayerRegistry() {} - - static string LayerTypeListString() { - vector layer_types = LayerTypeList(); - string layer_types_str; - for (vector::iterator iter = layer_types.begin(); - iter != layer_types.end(); ++iter) { - if (iter != layer_types.begin()) { - layer_types_str += ", "; - } - layer_types_str += *iter; - } - return layer_types_str; - } -}; + LayerRegistry(); + static string LayerTypeListString(); +}; template class LayerRegisterer { public: LayerRegisterer(const string& type, - shared_ptr > (*creator)(const LayerParameter&)) { - // LOG(INFO) << "Registering layer type: " << type; - LayerRegistry::AddCreator(type, creator); - } + shared_ptr > (*creator)(const LayerParameter&)); }; - #define REGISTER_LAYER_CREATOR(type, creator) \ static LayerRegisterer g_creator_f_##type(#type, creator); \ static LayerRegisterer g_creator_d_##type(#type, creator) \ diff --git a/include/caffe/solver_factory.hpp b/include/caffe/solver_factory.hpp index a5b160739b2..8c988018c43 100644 --- a/include/caffe/solver_factory.hpp +++ b/include/caffe/solver_factory.hpp @@ -56,69 +56,31 @@ class SolverRegistry { typedef Solver* (*Creator)(const SolverParameter&); typedef std::map CreatorRegistry; - static CreatorRegistry& Registry() { - static CreatorRegistry* g_registry_ = new CreatorRegistry(); - return *g_registry_; - } + static CreatorRegistry& Registry(); // Adds a creator. - static void AddCreator(const string& type, Creator creator) { - CreatorRegistry& registry = Registry(); - CHECK_EQ(registry.count(type), 0) - << "Solver type " << type << " already registered."; - registry[type] = creator; - } + static void AddCreator(const string& type, Creator creator); // Get a solver using a SolverParameter. - static Solver* CreateSolver(const SolverParameter& param) { - const string& type = param.type(); - CreatorRegistry& registry = Registry(); - CHECK_EQ(registry.count(type), 1) << "Unknown solver type: " << type - << " (known types: " << SolverTypeListString() << ")"; - return registry[type](param); - } - - static vector SolverTypeList() { - CreatorRegistry& registry = Registry(); - vector solver_types; - for (typename CreatorRegistry::iterator iter = registry.begin(); - iter != registry.end(); ++iter) { - solver_types.push_back(iter->first); - } - return solver_types; - } + static Solver* CreateSolver(const SolverParameter& param); + + static vector SolverTypeList(); private: // Solver registry should never be instantiated - everything is done with its // static variables. - SolverRegistry() {} - - static string SolverTypeListString() { - vector solver_types = SolverTypeList(); - string solver_types_str; - for (vector::iterator iter = solver_types.begin(); - iter != solver_types.end(); ++iter) { - if (iter != solver_types.begin()) { - solver_types_str += ", "; - } - solver_types_str += *iter; - } - return solver_types_str; - } -}; + SolverRegistry(); // {} + static string SolverTypeListString(); +}; template class SolverRegisterer { public: SolverRegisterer(const string& type, - Solver* (*creator)(const SolverParameter&)) { - // LOG(INFO) << "Registering solver type: " << type; - SolverRegistry::AddCreator(type, creator); - } + Solver* (*creator)(const SolverParameter&)); }; - #define REGISTER_SOLVER_CREATOR(type, creator) \ static SolverRegisterer g_creator_f_##type(#type, creator); \ static SolverRegisterer g_creator_d_##type(#type, creator) \ diff --git a/include/caffe/test/test_caffe_main.hpp b/include/caffe/test/test_caffe_main.hpp index 294f7e5011a..545e18d3d77 100644 --- a/include/caffe/test/test_caffe_main.hpp +++ b/include/caffe/test/test_caffe_main.hpp @@ -10,6 +10,7 @@ #include #include "caffe/common.hpp" +#include "caffe/util/io.hpp" using std::cout; using std::endl; @@ -34,7 +35,8 @@ class MultiDeviceTest : public ::testing::Test { MultiDeviceTest() { Caffe::set_mode(TypeParam::device); } - virtual ~MultiDeviceTest() {} + // Caffe tests may create some temporary files, here we will do the cleanup. + virtual ~MultiDeviceTest() { RemoveCaffeTempDir(); } }; typedef ::testing::Types TestDtypes; diff --git a/include/caffe/util/cudnn.hpp b/include/caffe/util/cudnn.hpp index 498cfe385de..3cd7dee8fcd 100644 --- a/include/caffe/util/cudnn.hpp +++ b/include/caffe/util/cudnn.hpp @@ -17,6 +17,7 @@ << cudnnGetErrorString(status); \ } while (0) +#if !defined (_MSC_VER) inline const char* cudnnGetErrorString(cudnnStatus_t status) { switch (status) { case CUDNN_STATUS_SUCCESS: @@ -48,6 +49,7 @@ inline const char* cudnnGetErrorString(cudnnStatus_t status) { } return "Unknown cudnn status"; } +#endif namespace caffe { diff --git a/include/caffe/util/io.hpp b/include/caffe/util/io.hpp index 1a599883ca3..6375a4e3be5 100644 --- a/include/caffe/util/io.hpp +++ b/include/caffe/util/io.hpp @@ -23,8 +23,12 @@ using ::boost::filesystem::path; inline void MakeTempDir(string* temp_dirname) { temp_dirname->clear(); - const path& model = - boost::filesystem::temp_directory_path()/"caffe_test.%%%%-%%%%"; + // Place all temp directories under temp_root, to be able to delete all of + // them at once, without knowing their name. + const path& temp_root = + boost::filesystem::temp_directory_path() / "caffe_test"; + boost::filesystem::create_directory(temp_root); + const path& model = temp_root / "%%%%-%%%%"; for ( int i = 0; i < CAFFE_TMP_DIR_RETRIES; i++ ) { const path& dir = boost::filesystem::unique_path(model).string(); bool done = boost::filesystem::create_directory(dir); @@ -37,7 +41,7 @@ inline void MakeTempDir(string* temp_dirname) { } inline void MakeTempFilename(string* temp_filename) { - static path temp_files_subpath; + path temp_files_subpath; static uint64_t next_temp_file = 0; temp_filename->clear(); if ( temp_files_subpath.empty() ) { @@ -49,6 +53,21 @@ inline void MakeTempFilename(string* temp_filename) { (temp_files_subpath/caffe::format_int(next_temp_file++, 9)).string(); } +#ifdef _MSC_VER + +inline void RemoveCaffeTempDir() { + boost::system::error_code err; + boost::filesystem::remove_all( + boost::filesystem::temp_directory_path() / "caffe_test", err); +} + +#else + +inline void RemoveCaffeTempDir() { +} + +#endif + bool ReadProtoFromTextFile(const char* filename, Message* proto); inline bool ReadProtoFromTextFile(const string& filename, Message* proto) { diff --git a/matlab/+caffe/+test/test_solver.m b/matlab/+caffe/+test/test_solver.m index 739258b0e85..bfd0c75f805 100644 --- a/matlab/+caffe/+test/test_solver.m +++ b/matlab/+caffe/+test/test_solver.m @@ -13,7 +13,7 @@ fid = fopen(solver_file, 'w'); fprintf(fid, [ ... - 'net: "' model_file '"\n' ... + 'net: "' strrep(model_file, '\', '/') '"\n' ... 'test_iter: 10 test_interval: 10 base_lr: 0.01 momentum: 0.9\n' ... 'weight_decay: 0.0005 lr_policy: "inv" gamma: 0.0001 power: 0.75\n' ... 'display: 100 max_iter: 100 snapshot_after_train: false\n' ]); diff --git a/matlab/+caffe/private/caffe_.cpp b/matlab/+caffe/private/caffe_.cpp index a32bd5e536d..32b66c3ae15 100644 --- a/matlab/+caffe/private/caffe_.cpp +++ b/matlab/+caffe/private/caffe_.cpp @@ -45,7 +45,14 @@ void mxCHECK_FILE_EXIST(const char* file) { static vector > > solvers_; static vector > > nets_; // init_key is generated at the beginning and every time you call reset +#ifndef _MSC_VER // We are not using MSVC. static double init_key = static_cast(caffe_rng_rand()); +#else // We are using MSVC. +// The original statement may cause MATLAB halt on Windows when cuBLAS is used. +// Using a negative number as a flag instead of calling caffe_rng_rand(). +// init_key will be generated in entry function: mexFunction(). +static double init_key = -1; +#endif // !_MSC_VER /** ----------------------------------------------------------------- ** data conversion functions @@ -583,6 +590,10 @@ static handler_registry handlers[] = { **/ // Usage: caffe_(api_command, arg1, arg2, ...) void mexFunction(MEX_ARGS) { +#ifdef _MSC_VER + if (init_key == -1) + init_key = static_cast(caffe_rng_rand()); +#endif // _MSC_VER mexLock(); // Avoid clearing the mex file. mxCHECK(nrhs > 0, "Usage: caffe_(api_command, arg1, arg2, ...)"); // Handle input command diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 987730d9b55..25f6a5cbef5 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -1,5 +1,5 @@ # Builds Matlab (or Octave) interface. In case of Matlab caffe must be -# compield as shared library. Octave can link static or shared caffe library +# compiled as shared library. Octave can link static or shared caffe library # To install octave run: sudo apt-get install liboctave-dev if(NOT BUILD_matlab) @@ -16,26 +16,34 @@ else() return() endif() -if(NOT BUILD_SHARED_LIBS AND build_using MATCHES Matlab) - message(FATAL_ERROR "Matlab MEX interface (with default mex options file) can only be built if caffe is compiled as shared library. Please enable 'BUILD_SHARED_LIBS' in CMake. Aternativelly you can switch to Octave compiler.") +if(NOT MSVC AND NOT BUILD_SHARED_LIBS AND build_using MATCHES Matlab) + message(FATAL_ERROR "Matlab MEX interface (with default mex options file) can only be built if caffe is compiled as shared library on UNIX systems. Please enable 'BUILD_SHARED_LIBS' in CMake. Alternatively you can switch to Octave compiler.") endif() +if(NOT MSVC) # helper function to set proper mex file extension -function(caffe_fetch_and_set_proper_mexext mexfile_variable) - execute_process(COMMAND ${Matlab_mexext} OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE res OUTPUT_VARIABLE ext) - if(res MATCHES 0) - get_filename_component(folder ${${mexfile_variable}} PATH) - get_filename_component(name_we ${${mexfile_variable}} NAME_WE) - set(${mexfile_variable} ${folder}/${name_we}.${ext} PARENT_SCOPE) - endif() -endfunction() + function(caffe_fetch_and_set_proper_mexext mexfile_variable) + execute_process(COMMAND ${Matlab_mexext} OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE res OUTPUT_VARIABLE ext) + if(res MATCHES 0) + get_filename_component(folder ${${mexfile_variable}} PATH) + get_filename_component(name_we ${${mexfile_variable}} NAME_WE) + set(${mexfile_variable} ${folder}/${name_we}.${ext} PARENT_SCOPE) + endif() + endfunction() +endif() # global settings file(GLOB Matlab_srcs +caffe/private/caffe_.cpp) -set(Matlab_caffe_mex ${PROJECT_SOURCE_DIR}/matlab/+caffe/private/caffe_.mex) +if(MSVC) + set(Matlab_caffe_mex ${PROJECT_SOURCE_DIR}/matlab/+caffe/private/caffe_.mexw64) +else() + set(Matlab_caffe_mex ${PROJECT_SOURCE_DIR}/matlab/+caffe/private/caffe_.mex) +endif() caffe_get_current_cflags(cflags) -caffe_parse_linker_libs(Caffe_LINKER_LIBS folders libflags macos_frameworks) +if(NOT MSVC) + caffe_parse_linker_libs(Caffe_LINKER_LIBS folders libflags macos_frameworks) +endif() set(folders $ ${folders}) # prepare linker flag lists @@ -43,14 +51,28 @@ string(REPLACE ";" ";-L" link_folders "-L${folders}") string(REPLACE ";" ":" rpath_folders "${folders}") if(build_using MATCHES "Matlab") - set(libflags -lcaffe${Caffe_POSTFIX} ${libflags}) # Matlab R2014a complans for -Wl,--whole-archive - - caffe_fetch_and_set_proper_mexext(Matlab_caffe_mex) - add_custom_command(OUTPUT ${Matlab_caffe_mex} COMMAND ${Matlab_mex} - ARGS -output ${Matlab_caffe_mex} ${Matlab_srcs} ${cflags} ${link_folders} ${libflags} - DEPENDS caffe COMMENT "Building Matlab interface: ${Matlab_caffe_mex}" VERBATIM) - add_custom_target(matlab ALL DEPENDS ${Matlab_caffe_mex} SOURCES ${Matlab_srcs}) + if(MSVC) + matlab_add_mex(NAME matlab + SRC ${Matlab_srcs} # maybe you need to add some other sources + OUTPUT_NAME caffe_ # change the output name to _caffe.mexw64 + LINK_TO caffe # cmake will take care of forwarding the correct transitive library dependencies to your mex file + ) + # output the target in the source tree as in the original version. + set_target_properties(matlab PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/matlab/+caffe/private + ) + if(COPY_PREREQUISITES) + caffe_copy_prerequisites(matlab DESTINATION ${PROJECT_SOURCE_DIR}/matlab/+caffe/private USE_HARD_LINKS) + endif() + else() + set(libflags -lcaffe${Caffe_POSTFIX} ${libflags}) # Matlab R2014a complans for -Wl,--whole-archive + caffe_fetch_and_set_proper_mexext(Matlab_caffe_mex) + add_custom_command(OUTPUT ${Matlab_caffe_mex} COMMAND ${Matlab_mex} + ARGS -output ${Matlab_caffe_mex} ${Matlab_srcs} ${cflags} ${link_folders} ${libflags} + DEPENDS caffe COMMENT "Building Matlab interface: ${Matlab_caffe_mex}" VERBATIM) + add_custom_target(matlab ALL DEPENDS ${Matlab_caffe_mex} SOURCES ${Matlab_srcs}) + endif() elseif(build_using MATCHES "Octave") if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") @@ -67,6 +89,16 @@ elseif(build_using MATCHES "Octave") endif() # ---[ Install -file(GLOB mfiles caffe/*.m) -install(FILES ${mfiles} ${Matlab_caffe_mex} DESTINATION matlab) +if(MSVC) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/matlab DESTINATION . + PATTERN CMakeLists.txt EXCLUDE + PATTERN .gitignore EXCLUDE) +else() + file(GLOB mfiles caffe/*.m) + install(FILES ${mfiles} ${Matlab_caffe_mex} DESTINATION matlab) +endif() + +if(MSVC AND INSTALL_PREREQUISITES) + caffe_install_prerequisites(matlab DESTINATION matlab/+caffe/private) +endif() diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index c53299d265b..00fb51bbdcf 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -5,9 +5,12 @@ endif() file(GLOB_RECURSE python_srcs ${PROJECT_SOURCE_DIR}/python/*.cpp) -add_library(pycaffe SHARED ${python_srcs}) +add_library(pycaffe MODULE ${python_srcs}) caffe_default_properties(pycaffe) set_target_properties(pycaffe PROPERTIES PREFIX "" OUTPUT_NAME "_caffe") +if(MSVC) + set_target_properties(pycaffe PROPERTIES SUFFIX ".pyd") +endif() target_include_directories(pycaffe PUBLIC ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR}) target_link_libraries(pycaffe PUBLIC ${Caffe_LINK} ${PYTHON_LIBRARIES}) @@ -19,6 +22,17 @@ if(UNIX OR APPLE) COMMAND touch ${PROJECT_SOURCE_DIR}/python/caffe/proto/__init__.py COMMAND cp ${proto_gen_folder}/*.py ${PROJECT_SOURCE_DIR}/python/caffe/proto/ COMMENT "Creating symlink ${__linkname} -> ${PROJECT_BINARY_DIR}/lib/_caffe${Caffe_POSTFIX}.so") +elseif(WIN32) + add_custom_command(TARGET pycaffe POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${PROJECT_SOURCE_DIR}/python/caffe + COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_SOURCE_DIR}/python/caffe/proto + COMMAND ${CMAKE_COMMAND} -E touch ${PROJECT_SOURCE_DIR}/python/caffe/proto/__init__.py + COMMAND (robocopy "\"${proto_gen_folder}\" \"${PROJECT_SOURCE_DIR}/python/caffe/proto\" *.py") ^& IF %ERRORLEVEL% LEQ 4 set ERRORLEVEL=0 + COMMENT "Creating symlink ${__linkname} -> ${PROJECT_BINARY_DIR}/lib/_caffe.pyd") +endif() + +if(MSVC AND COPY_PREREQUISITES) + caffe_copy_prerequisites(pycaffe DESTINATION ${PROJECT_SOURCE_DIR}/python/caffe USE_HARD_LINKS) endif() # ---[ Install @@ -38,3 +52,8 @@ install(DIRECTORY caffe # _caffe.so install(TARGETS pycaffe DESTINATION python/caffe) +if(MSVC AND INSTALL_PREREQUISITES) + caffe_install_prerequisites(pycaffe DESTINATION python/caffe) +endif() + + diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp index d7f43fff62d..8e4a3ce099b 100644 --- a/python/caffe/_caffe.cpp +++ b/python/caffe/_caffe.cpp @@ -1,5 +1,3 @@ -#include // NOLINT(build/include_alpha) - // Produce deprecation warnings (needs to come before arrayobject.h inclusion). #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION @@ -39,6 +37,41 @@ } \ } while (0) +#if defined(_MSC_VER) && (_MSC_FULL_VER >= 190024210) +// Workaround for VS 2015 Update 3 which breaks boost python +// See: http://stackoverflow.com/questions/38261530/unresolved-external-symbols-since-visual-studio-2015-update-3-boost-python-link +// and https://msdn.microsoft.com/vs-knownissues/vs2015-update3 +#define BP_GET_POINTER(cls) \ +namespace boost { \ +template <> \ +const volatile caffe::cls * \ +get_pointer(const volatile caffe::cls *c) { \ + return c; \ +} \ +} + +#define BP_GET_POINTER_T(cls, dtype) BP_GET_POINTER(cls) + +// forward declare the NCCL class +// in case we are not using NCCL +namespace caffe { +template class NCCL; +} + +BP_GET_POINTER_T(Net, float); +BP_GET_POINTER_T(Layer, float); +BP_GET_POINTER_T(Solver, float); +BP_GET_POINTER_T(SGDSolver, float); +BP_GET_POINTER_T(NesterovSolver, float); +BP_GET_POINTER_T(AdaGradSolver, float); +BP_GET_POINTER_T(RMSPropSolver, float); +BP_GET_POINTER_T(AdaDeltaSolver, float); +BP_GET_POINTER_T(AdamSolver, float); +BP_GET_POINTER_T(NCCL, float); +BP_GET_POINTER(Timer); + +#endif + namespace bp = boost::python; namespace caffe { @@ -53,15 +86,18 @@ void set_mode_gpu() { Caffe::set_mode(Caffe::GPU); } void InitLog() { ::google::InitGoogleLogging(""); +#ifndef _MSC_VER + // this symbol is undefined on windows ::google::InstallFailureSignalHandler(); +#endif // _MSC_VER } void InitLogLevel(int level) { FLAGS_minloglevel = level; InitLog(); } -void InitLogLevelPipe(int level, bool stderr) { +void InitLogLevelPipe(int level, bool std_err) { FLAGS_minloglevel = level; - FLAGS_logtostderr = stderr; + FLAGS_logtostderr = std_err; InitLog(); } void Log(const string& s) { diff --git a/python/caffe/draw.py b/python/caffe/draw.py index 8411a41d1d4..1e1aec0b925 100644 --- a/python/caffe/draw.py +++ b/python/caffe/draw.py @@ -9,6 +9,7 @@ Caffe. """ +import os from caffe.proto import caffe_pb2 """ @@ -21,6 +22,35 @@ except ImportError: import pydot + +if os.name == 'nt': + # Workaround to find graphviz executables + # with graphviz conda package under windows + + # Monkeypatch the pydot package + pydot_find_graphviz = pydot.graphviz.find_graphviz + + def resolve_graphviz_executables(): + """ + Resolve the graphviz executables by adding a `graphviz` suffix + to folders located on path + """ + # first check if we can find the executables the normal way + progs = pydot_find_graphviz() + if not progs: + directories = os.environ['PATH'].split(';') + suffix = 'graphviz' + progs = {} + for directory in directories: + for exe in ['dot', 'twopi', 'neato', 'circo', 'fdp']: + full_path = os.path.join(directory, suffix, + '{}.exe'.format(exe)) + if os.path.exists(full_path): + progs[exe] = full_path + return progs + + pydot.graphviz.find_graphviz = resolve_graphviz_executables + # Internal layer and blob styles. LAYER_STYLE_DEFAULT = {'shape': 'record', 'fillcolor': '#6495ED', diff --git a/python/caffe/test/test_python_layer.py b/python/caffe/test/test_python_layer.py index 899514e90f1..9700edc521d 100644 --- a/python/caffe/test/test_python_layer.py +++ b/python/caffe/test/test_python_layer.py @@ -151,6 +151,10 @@ def test_parameter(self): self.assertEqual(layer.blobs[0].data[0], -1) net.copy_from(caffemodel_file) self.assertEqual(layer.blobs[0].data[0], 1) + if os.name == 'nt': + # On Windows, attempting to remove a file that is in use + # causes an exception to be raised." + os.close(h) os.remove(caffemodel_file) # Test weight sharing diff --git a/python/caffe/test/test_solver.py b/python/caffe/test/test_solver.py index f618fded8cd..4c1f09666b0 100644 --- a/python/caffe/test/test_solver.py +++ b/python/caffe/test/test_solver.py @@ -13,7 +13,10 @@ def setUp(self): self.num_output = 13 net_f = simple_net_file(self.num_output) f = tempfile.NamedTemporaryFile(mode='w+', delete=False) - f.write("""net: '""" + net_f + """' + net_f_mod = net_f + if os.name == 'nt': + net_f_mod = net_f_mod.replace("\\", "/") + f.write("""net: '""" + net_f_mod + """' test_iter: 10 test_interval: 10 base_lr: 0.01 momentum: 0.9 weight_decay: 0.0005 lr_policy: 'inv' gamma: 0.0001 power: 0.75 display: 100 max_iter: 100 snapshot_after_train: false diff --git a/scripts/appveyor/appveyor_install_cuda.cmd b/scripts/appveyor/appveyor_install_cuda.cmd new file mode 100644 index 00000000000..722e32b22db --- /dev/null +++ b/scripts/appveyor/appveyor_install_cuda.cmd @@ -0,0 +1,14 @@ +@echo off +echo Downloading CUDA toolkit 8 ... +appveyor DownloadFile https://developer.nvidia.com/compute/cuda/8.0/prod/local_installers/cuda_8.0.44_windows-exe -FileName cuda_8.0.44_windows.exe +echo Installing CUDA toolkit 8 ... +cuda_8.0.44_windows.exe -s compiler_8.0 ^ + cublas_8.0 ^ + cublas_dev_8.0 ^ + cudart_8.0 ^ + curand_8.0 ^ + curand_dev_8.0 ^ + nvml_dev_8.0 +:: Add CUDA toolkit to PATH +set PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v8.0\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v8.0\libnvvp;%PATH% +nvcc -V \ No newline at end of file diff --git a/scripts/build_win.cmd b/scripts/build_win.cmd new file mode 100644 index 00000000000..06f06fababf --- /dev/null +++ b/scripts/build_win.cmd @@ -0,0 +1,228 @@ +@echo off +@setlocal EnableDelayedExpansion + +:: Default values +if DEFINED APPVEYOR ( + echo Setting Appveyor defaults + if NOT DEFINED MSVC_VERSION set MSVC_VERSION=14 + if NOT DEFINED WITH_NINJA set WITH_NINJA=1 + if NOT DEFINED CPU_ONLY set CPU_ONLY=1 + if NOT DEFINED CUDA_ARCH_NAME set CUDA_ARCH_NAME=Auto + if NOT DEFINED CMAKE_CONFIG set CMAKE_CONFIG=Release + if NOT DEFINED USE_NCCL set USE_NCCL=0 + if NOT DEFINED CMAKE_BUILD_SHARED_LIBS set CMAKE_BUILD_SHARED_LIBS=0 + if NOT DEFINED PYTHON_VERSION set PYTHON_VERSION=2 + if NOT DEFINED BUILD_PYTHON set BUILD_PYTHON=1 + if NOT DEFINED BUILD_PYTHON_LAYER set BUILD_PYTHON_LAYER=1 + if NOT DEFINED BUILD_MATLAB set BUILD_MATLAB=0 + if NOT DEFINED PYTHON_EXE set PYTHON_EXE=python + if NOT DEFINED RUN_TESTS set RUN_TESTS=1 + if NOT DEFINED RUN_LINT set RUN_LINT=1 + if NOT DEFINED RUN_INSTALL set RUN_INSTALL=1 + + :: Set python 2.7 with conda as the default python + if !PYTHON_VERSION! EQU 2 ( + set CONDA_ROOT=C:\Miniconda-x64 + ) + :: Set python 3.5 with conda as the default python + if !PYTHON_VERSION! EQU 3 ( + set CONDA_ROOT=C:\Miniconda35-x64 + ) + set PATH=!CONDA_ROOT!;!CONDA_ROOT!\Scripts;!CONDA_ROOT!\Library\bin;!PATH! + + :: Check that we have the right python version + !PYTHON_EXE! --version + :: Add the required channels + conda config --add channels conda-forge + conda config --add channels willyd + :: Update conda + conda update conda -y + :: Download other required packages + conda install --yes cmake ninja numpy scipy protobuf==3.1.0 six scikit-image pyyaml pydotplus graphviz + + if ERRORLEVEL 1 ( + echo ERROR: Conda update or install failed + exit /b 1 + ) + + :: Install cuda and disable tests if needed + if !WITH_CUDA! == 1 ( + call %~dp0\appveyor\appveyor_install_cuda.cmd + set CPU_ONLY=0 + set RUN_TESTS=0 + set USE_NCCL=1 + ) else ( + set CPU_ONLY=1 + ) + + :: Disable the tests in debug config + if "%CMAKE_CONFIG%" == "Debug" ( + echo Disabling tests on appveyor with config == %CMAKE_CONFIG% + set RUN_TESTS=0 + ) + + :: Disable linting with python 3 until we find why the script fails + if !PYTHON_VERSION! EQU 3 ( + set RUN_LINT=0 + ) + +) else ( + :: Change the settings here to match your setup + :: Change MSVC_VERSION to 12 to use VS 2013 + if NOT DEFINED MSVC_VERSION set MSVC_VERSION=14 + :: Change to 1 to use Ninja generator (builds much faster) + if NOT DEFINED WITH_NINJA set WITH_NINJA=1 + :: Change to 1 to build caffe without CUDA support + if NOT DEFINED CPU_ONLY set CPU_ONLY=0 + :: Change to generate CUDA code for one of the following GPU architectures + :: [Fermi Kepler Maxwell Pascal All] + if NOT DEFINED CUDA_ARCH_NAME set CUDA_ARCH_NAME=Auto + :: Change to Debug to build Debug. This is only relevant for the Ninja generator the Visual Studio generator will generate both Debug and Release configs + if NOT DEFINED CMAKE_CONFIG set CMAKE_CONFIG=Release + :: Set to 1 to use NCCL + if NOT DEFINED USE_NCCL set USE_NCCL=0 + :: Change to 1 to build a caffe.dll + if NOT DEFINED CMAKE_BUILD_SHARED_LIBS set CMAKE_BUILD_SHARED_LIBS=0 + :: Change to 3 if using python 3.5 (only 2.7 and 3.5 are supported) + if NOT DEFINED PYTHON_VERSION set PYTHON_VERSION=2 + :: Change these options for your needs. + if NOT DEFINED BUILD_PYTHON set BUILD_PYTHON=1 + if NOT DEFINED BUILD_PYTHON_LAYER set BUILD_PYTHON_LAYER=1 + if NOT DEFINED BUILD_MATLAB set BUILD_MATLAB=0 + :: If python is on your path leave this alone + if NOT DEFINED PYTHON_EXE set PYTHON_EXE=python + :: Run the tests + if NOT DEFINED RUN_TESTS set RUN_TESTS=0 + :: Run lint + if NOT DEFINED RUN_LINT set RUN_LINT=0 + :: Build the install target + if NOT DEFINED RUN_INSTALL set RUN_INSTALL=0 +) + +:: Set the appropriate CMake generator +:: Use the exclamation mark ! below to delay the +:: expansion of CMAKE_GENERATOR +if %WITH_NINJA% EQU 0 ( + if "%MSVC_VERSION%"=="14" ( + set CMAKE_GENERATOR=Visual Studio 14 2015 Win64 + ) + if "%MSVC_VERSION%"=="12" ( + set CMAKE_GENERATOR=Visual Studio 12 2013 Win64 + ) + if "!CMAKE_GENERATOR!"=="" ( + echo ERROR: Unsupported MSVC version + exit /B 1 + ) +) else ( + set CMAKE_GENERATOR=Ninja +) + +echo INFO: ============================================================ +echo INFO: Summary: +echo INFO: ============================================================ +echo INFO: MSVC_VERSION = !MSVC_VERSION! +echo INFO: WITH_NINJA = !WITH_NINJA! +echo INFO: CMAKE_GENERATOR = "!CMAKE_GENERATOR!" +echo INFO: CPU_ONLY = !CPU_ONLY! +echo INFO: CUDA_ARCH_NAME = !CUDA_ARCH_NAME! +echo INFO: CMAKE_CONFIG = !CMAKE_CONFIG! +echo INFO: USE_NCCL = !USE_NCCL! +echo INFO: CMAKE_BUILD_SHARED_LIBS = !CMAKE_BUILD_SHARED_LIBS! +echo INFO: PYTHON_VERSION = !PYTHON_VERSION! +echo INFO: BUILD_PYTHON = !BUILD_PYTHON! +echo INFO: BUILD_PYTHON_LAYER = !BUILD_PYTHON_LAYER! +echo INFO: BUILD_MATLAB = !BUILD_MATLAB! +echo INFO: PYTHON_EXE = "!PYTHON_EXE!" +echo INFO: RUN_TESTS = !RUN_TESTS! +echo INFO: RUN_LINT = !RUN_LINT! +echo INFO: RUN_INSTALL = !RUN_INSTALL! +echo INFO: ============================================================ + +:: Build and exectute the tests +:: Do not run the tests with shared library +if !RUN_TESTS! EQU 1 ( + if %CMAKE_BUILD_SHARED_LIBS% EQU 1 ( + echo WARNING: Disabling tests with shared library build + set RUN_TESTS=0 + ) +) + +if NOT EXIST build mkdir build +pushd build + +:: Setup the environement for VS x64 +set batch_file=!VS%MSVC_VERSION%0COMNTOOLS!..\..\VC\vcvarsall.bat +call "%batch_file%" amd64 + +:: Configure using cmake and using the caffe-builder dependencies +:: Add -DCUDNN_ROOT=C:/Projects/caffe/cudnn-8.0-windows10-x64-v5.1/cuda ^ +:: below to use cuDNN +cmake -G"!CMAKE_GENERATOR!" ^ + -DBLAS=Open ^ + -DCMAKE_BUILD_TYPE:STRING=%CMAKE_CONFIG% ^ + -DBUILD_SHARED_LIBS:BOOL=%CMAKE_BUILD_SHARED_LIBS% ^ + -DBUILD_python:BOOL=%BUILD_PYTHON% ^ + -DBUILD_python_layer:BOOL=%BUILD_PYTHON_LAYER% ^ + -DBUILD_matlab:BOOL=%BUILD_MATLAB% ^ + -DCPU_ONLY:BOOL=%CPU_ONLY% ^ + -DCOPY_PREREQUISITES:BOOL=1 ^ + -DINSTALL_PREREQUISITES:BOOL=1 ^ + -DUSE_NCCL:BOOL=!USE_NCCL! ^ + -DCUDA_ARCH_NAME:STRING=%CUDA_ARCH_NAME% ^ + "%~dp0\.." + +if ERRORLEVEL 1 ( + echo ERROR: Configure failed + exit /b 1 +) + +:: Lint +if %RUN_LINT% EQU 1 ( + cmake --build . --target lint --config %CMAKE_CONFIG% +) + +if ERRORLEVEL 1 ( + echo ERROR: Lint failed + exit /b 1 +) + +:: Build the library and tools +cmake --build . --config %CMAKE_CONFIG% + +if ERRORLEVEL 1 ( + echo ERROR: Build failed + exit /b 1 +) + +:: Build and exectute the tests +if !RUN_TESTS! EQU 1 ( + cmake --build . --target runtest --config %CMAKE_CONFIG% + + if ERRORLEVEL 1 ( + echo ERROR: Tests failed + exit /b 1 + ) + + if %BUILD_PYTHON% EQU 1 ( + if %BUILD_PYTHON_LAYER% EQU 1 ( + :: Run python tests only in Release build since + :: the _caffe module is _caffe-d is debug + if "%CMAKE_CONFIG%"=="Release" ( + :: Run the python tests + cmake --build . --target pytest + + if ERRORLEVEL 1 ( + echo ERROR: Python tests failed + exit /b 1 + ) + ) + ) + ) +) + +if %RUN_INSTALL% EQU 1 ( + cmake --build . --target install --config %CMAKE_CONFIG% +) + +popd +@endlocal \ No newline at end of file diff --git a/scripts/download_prebuilt_dependencies.py b/scripts/download_prebuilt_dependencies.py new file mode 100644 index 00000000000..01000f17575 --- /dev/null +++ b/scripts/download_prebuilt_dependencies.py @@ -0,0 +1,60 @@ +#!/usr/bin/python +# +# copyright Guillaume Dumont (2016) + +import os +import sys +import hashlib +import argparse +import tarfile + +from six.moves import urllib +from download_model_binary import reporthook + +WIN_DEPENDENCIES_URLS = { + ('v120', '2.7'):("https://github.com/willyd/caffe-builder/releases/download/v1.1.0/libraries_v120_x64_py27_1.1.0.tar.bz2", + "ba833d86d19b162a04d68b09b06df5e0dad947d4"), + ('v140', '2.7'):("https://github.com/willyd/caffe-builder/releases/download/v1.1.0/libraries_v140_x64_py27_1.1.0.tar.bz2", + "17eecb095bd3b0774a87a38624a77ce35e497cd2"), + ('v140', '3.5'):("https://github.com/willyd/caffe-builder/releases/download/v1.1.0/libraries_v140_x64_py35_1.1.0.tar.bz2", + "f060403fd1a7448d866d27c0e5b7dced39c0a607"), +} + +# function for checking SHA1. +def model_checks_out(filename, sha1): + with open(filename, 'rb') as f: + return hashlib.sha1(f.read()).hexdigest() == sha1 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Download prebuilt dependencies for windows.') + parser.add_argument('--msvc_version', default='v120', choices=['v120', 'v140']) + args = parser.parse_args() + + # get the appropriate url + pyver = '{:d}.{:d}'.format(sys.version_info.major, sys.version_info.minor) + try: + url, sha1 = WIN_DEPENDENCIES_URLS[(args.msvc_version, pyver)] + except KeyError: + print('ERROR: Could not find url for MSVC version = {} and Python version = {}.\n{}' + .format(args.msvc_version, pyver, + 'Available combinations are: {}'.format(list(WIN_DEPENDENCIES_URLS.keys())))) + sys.exit(1) + + dep_filename = os.path.split(url)[1] + # Download binaries + print("Downloading dependencies ({}). Please wait...".format(dep_filename)) + urllib.request.urlretrieve(url, dep_filename, reporthook) + if not model_checks_out(dep_filename, sha1): + print('ERROR: dependencies did not download correctly! Run this again.') + sys.exit(1) + print("\nDone.") + + # Extract the binaries from the tar file + tar = tarfile.open(dep_filename, 'r:bz2') + print("Extracting dependencies. Please wait...") + tar.extractall() + print("Done.") + tar.close() + diff --git a/src/caffe/CMakeLists.txt b/src/caffe/CMakeLists.txt index 4a805568566..313247703ee 100644 --- a/src/caffe/CMakeLists.txt +++ b/src/caffe/CMakeLists.txt @@ -15,8 +15,41 @@ list(INSERT Caffe_LINKER_LIBS 0 PUBLIC caffeproto) # note, crucial to prepend! # creates 'test_srcs', 'srcs', 'test_cuda', 'cuda' lists caffe_pickup_caffe_sources(${PROJECT_SOURCE_DIR}) +# add this option here since CUDA will not honor +# target_compile_definitions +if(MSVC AND NOT BUILD_SHARED_LIBS) + set(_caffe_static_compile_def -DCAFFE_BUILDING_STATIC_LIB) +endif() + if(HAVE_CUDA) + # collect any compile definitions from imported targets. This important so that + # preprocessor macros such as GLOG_NO_ABBREVIATED_SEVERITIES are defined. + # this is required since CUDA macros do not honor the INTERFACE_COMPILE_DEFINITIONS + unset(__cuda_options) + foreach(__lib ${Caffe_LINKER_LIBS}) + if(TARGET ${__lib}) + get_target_property(__interface_compile_definitions ${__lib} INTERFACE_COMPILE_DEFINITIONS) + if(__interface_compile_definitions) + foreach(__def ${__interface_compile_definitions}) + # espace any parentheses because they are failing the build + # see cmake issue https://cmake.org/Bug/view.php?id=16065 + string(REPLACE "(" "\\\(" __def_escaped ${__def}) + string(REPLACE ")" "\\\)" __def_escaped ${__def_escaped}) + # add the required -D flag + list(APPEND __cuda_options "-D${__def_escaped}") + endforeach() + endif() + endif() + endforeach() + list(APPEND __cuda_options ${_caffe_static_compile_def}) + # add the required definitions + add_definitions(${__cuda_options}) + # it seems that using the OPTIONS argument like: + # caffe_cuda_compile(cuda_objs ${cuda} OPTIONS ${__cuda_options}) + # does not work. Use add/remove_definitions instead. caffe_cuda_compile(cuda_objs ${cuda}) + # remove them + remove_definitions(${__cuda_options}) list(APPEND srcs ${cuda_objs} ${cuda}) endif() @@ -35,6 +68,33 @@ set_target_properties(caffe PROPERTIES VERSION ${CAFFE_TARGET_VERSION} SOVERSION ${CAFFE_TARGET_SOVERSION} ) +if(MSVC AND BUILD_SHARED_LIBS) + # CMake 3.4 introduced a WINDOWS_EXPORT_ALL_SYMBOLS target property that makes it possible to + # build shared libraries without using the usual declspec() decoration. + # See: https://blog.kitware.com/create-dlls-on-windows-without-declspec-using-new-cmake-export-all-feature/ + # and https://cmake.org/cmake/help/v3.5/prop_tgt/WINDOWS_EXPORT_ALL_SYMBOLS.html + # for details. + set_target_properties(caffe PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) +elseif(MSVC AND NOT BUILD_SHARED_LIBS) + # add a custom build command that generates a list of symbols + # to force linking. This is required because MSVC as nothing + # the whole-archive option + windows_create_link_header(caffe ${caffe_symbols_hdr}) + get_filename_component(_name ${caffe_symbols_hdr} NAME) + set(CAFFE_INCLUDE_SYMBOLS "#include \"caffe/${_name}\"") + # definition needed to include CMake generated files + target_compile_definitions(caffe PRIVATE ${_caffe_static_compile_def} + PUBLIC -DCMAKE_WINDOWS_BUILD) +endif() +if(MSVC) + # Disable Boost autolinking for consuming projects + target_compile_definitions(caffe PUBLIC -DBOOST_ALL_NO_LIB) +endif() +if(MSVC AND USE_NCCL) + add_dependencies(caffe nccl) +endif() + +configure_file(${caffe_export_hdr_in} ${caffe_export_hdr}) # ---[ Tests add_subdirectory(test) @@ -43,6 +103,9 @@ set_target_properties(caffe PROPERTIES install(DIRECTORY ${Caffe_INCLUDE_DIR}/caffe DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install(FILES ${proto_hdrs} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/caffe/proto) install(TARGETS caffe caffeproto EXPORT CaffeTargets DESTINATION ${CMAKE_INSTALL_LIBDIR}) +if(MSVC AND NOT BUILD_SHARED_LIBS) + install(FILES ${caffe_export_hdr} ${caffe_symbols_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/caffe) +endif() file(WRITE ${PROJECT_BINARY_DIR}/__init__.py) list(APPEND proto_python ${PROJECT_BINARY_DIR}/__init__.py) diff --git a/src/caffe/common.cpp b/src/caffe/common.cpp index 4f6f9bccc38..a771513ae7b 100644 --- a/src/caffe/common.cpp +++ b/src/caffe/common.cpp @@ -1,3 +1,8 @@ +#if defined(_MSC_VER) +#include +#define getpid() _getpid() +#endif + #include #include #include @@ -46,7 +51,11 @@ void GlobalInit(int* pargc, char*** pargv) { // Google logging. ::google::InitGoogleLogging(*(pargv)[0]); // Provide a backtrace on segfault. + + // Windows port of glogs doesn't have this function built +#if !defined(_MSC_VER) ::google::InstallFailureSignalHandler(); +#endif } #ifdef CPU_ONLY // CPU-only Caffe. diff --git a/src/caffe/layer_factory.cpp b/src/caffe/layer_factory.cpp index f14253a510e..41dee3de046 100644 --- a/src/caffe/layer_factory.cpp +++ b/src/caffe/layer_factory.cpp @@ -4,6 +4,7 @@ #include #endif #include +#include #include "caffe/layer.hpp" #include "caffe/layer_factory.hpp" @@ -33,6 +34,78 @@ namespace caffe { +template +typename LayerRegistry::CreatorRegistry& +LayerRegistry::Registry() { + static CreatorRegistry* g_registry_ = new CreatorRegistry(); + return *g_registry_; +} + +// Adds a creator. +template +void LayerRegistry::AddCreator(const string& type, Creator creator) { + CreatorRegistry& registry = Registry(); + CHECK_EQ(registry.count(type), 0) << "Layer type " << type + << " already registered."; + registry[type] = creator; +} + +// Get a layer using a LayerParameter. +template +shared_ptr > LayerRegistry::CreateLayer( + const LayerParameter& param) { + if (Caffe::root_solver()) { + LOG(INFO) << "Creating layer " << param.name(); + } + const string& type = param.type(); + CreatorRegistry& registry = Registry(); + CHECK_EQ(registry.count(type), 1) + << "Unknown layer type: " << type + << " (known types: " << LayerTypeListString() << ")"; + return registry[type](param); +} + +template +vector LayerRegistry::LayerTypeList() { + CreatorRegistry& registry = Registry(); + vector layer_types; + for (typename CreatorRegistry::iterator iter = registry.begin(); + iter != registry.end(); ++iter) { + layer_types.push_back(iter->first); + } + return layer_types; +} + +// Layer registry should never be instantiated - everything is done with its +// static variables. +template +LayerRegistry::LayerRegistry() {} + +template +string LayerRegistry::LayerTypeListString() { + vector layer_types = LayerTypeList(); + string layer_types_str; + for (vector::iterator iter = layer_types.begin(); + iter != layer_types.end(); ++iter) { + if (iter != layer_types.begin()) { + layer_types_str += ", "; + } + layer_types_str += *iter; + } + return layer_types_str; +} + +template +LayerRegisterer::LayerRegisterer( + const string& type, + shared_ptr > (*creator)(const LayerParameter&)) { + // LOG(INFO) << "Registering layer type: " << type; + LayerRegistry::AddCreator(type, creator); +} + +INSTANTIATE_CLASS(LayerRegistry); +INSTANTIATE_CLASS(LayerRegisterer); + // Get convolution layer according to engine. template shared_ptr > GetConvolutionLayer( diff --git a/src/caffe/layers/bnll_layer.cu b/src/caffe/layers/bnll_layer.cu index 8df8ef09afe..768a92bba26 100644 --- a/src/caffe/layers/bnll_layer.cu +++ b/src/caffe/layers/bnll_layer.cu @@ -5,7 +5,7 @@ namespace caffe { -const float kBNLL_THRESHOLD = 50.; +__constant__ float kBNLL_THRESHOLD = 50.; template __global__ void BNLLForward(const int n, const Dtype* in, Dtype* out) { diff --git a/src/caffe/solver_factory.cpp b/src/caffe/solver_factory.cpp new file mode 100644 index 00000000000..8ee74546fd7 --- /dev/null +++ b/src/caffe/solver_factory.cpp @@ -0,0 +1,74 @@ +#include +#include + +#include "caffe/solver_factory.hpp" + +namespace caffe { + +template +typename SolverRegistry::CreatorRegistry& +SolverRegistry::Registry() { + static CreatorRegistry* g_registry_ = new CreatorRegistry; + return *g_registry_; +} + +template +void SolverRegistry::AddCreator(const string& type, Creator creator) { + CreatorRegistry& registry = Registry(); + CHECK_EQ(registry.count(type), 0) << "Solver type " << type + << " already registered."; + registry[type] = creator; +} + +// Get a solver using a SolverParameter. +template +Solver* SolverRegistry::CreateSolver( + const SolverParameter& param) { + const string& type = param.type(); + CreatorRegistry& registry = Registry(); + CHECK_EQ(registry.count(type), 1) + << "Unknown solver type: " << type + << " (known types: " << SolverTypeListString() << ")"; + return registry[type](param); +} + +template +vector SolverRegistry::SolverTypeList() { + CreatorRegistry& registry = Registry(); + vector solver_types; + for (typename CreatorRegistry::iterator iter = registry.begin(); + iter != registry.end(); ++iter) { + solver_types.push_back(iter->first); + } + return solver_types; +} + +// Solver registry should never be instantiated - everything is done with its +// static variables. +template +SolverRegistry::SolverRegistry() {} + +template +string SolverRegistry::SolverTypeListString() { + vector solver_types = SolverTypeList(); + string solver_types_str; + for (vector::iterator iter = solver_types.begin(); + iter != solver_types.end(); ++iter) { + if (iter != solver_types.begin()) { + solver_types_str += ", "; + } + solver_types_str += *iter; + } + return solver_types_str; +} + +template +SolverRegisterer::SolverRegisterer( + const string& type, Solver* (*creator)(const SolverParameter&)) { + SolverRegistry::AddCreator(type, creator); +} + +INSTANTIATE_CLASS(SolverRegistry); +INSTANTIATE_CLASS(SolverRegisterer); + +} // namespace caffe diff --git a/src/caffe/test/CMakeLists.txt b/src/caffe/test/CMakeLists.txt index d8afc30b76b..4a6ad64655f 100644 --- a/src/caffe/test/CMakeLists.txt +++ b/src/caffe/test/CMakeLists.txt @@ -31,6 +31,10 @@ target_link_libraries(${the_target} gtest ${Caffe_LINK}) caffe_default_properties(${the_target}) caffe_set_runtime_directory(${the_target} "${PROJECT_BINARY_DIR}/test") +if(MSVC AND COPY_PREREQUISITES) + caffe_copy_prerequisites(${the_target} USE_HARD_LINKS) +endif() + # ---[ Adding runtest add_custom_target(runtest COMMAND ${the_target} ${test_args} - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) \ No newline at end of file diff --git a/src/caffe/test/test_benchmark.cpp b/src/caffe/test/test_benchmark.cpp index b03fdf69a8a..96cd1bd0ba9 100644 --- a/src/caffe/test/test_benchmark.cpp +++ b/src/caffe/test/test_benchmark.cpp @@ -9,7 +9,12 @@ namespace caffe { +#ifdef _MSC_VER +// Timer tests have issues on appveyor +const float kMillisecondsThreshold = 50; +#else const float kMillisecondsThreshold = 30; +#endif template class BenchmarkTest : public MultiDeviceTest {}; diff --git a/src/caffe/test/test_blob.cpp b/src/caffe/test/test_blob.cpp index b88562223d0..4990475bb2a 100644 --- a/src/caffe/test/test_blob.cpp +++ b/src/caffe/test/test_blob.cpp @@ -35,12 +35,14 @@ TYPED_TEST(BlobSimpleTest, TestInitialization) { EXPECT_EQ(this->blob_->count(), 0); } +#if !defined(CPU_ONLY) TYPED_TEST(BlobSimpleTest, TestPointersCPUGPU) { EXPECT_TRUE(this->blob_preshaped_->gpu_data()); EXPECT_TRUE(this->blob_preshaped_->cpu_data()); EXPECT_TRUE(this->blob_preshaped_->mutable_gpu_data()); EXPECT_TRUE(this->blob_preshaped_->mutable_cpu_data()); } +#endif TYPED_TEST(BlobSimpleTest, TestReshape) { this->blob_->Reshape(2, 3, 4, 5); diff --git a/src/caffe/test/test_gradient_based_solver.cpp b/src/caffe/test/test_gradient_based_solver.cpp index f4395f5311c..85ac13688fb 100644 --- a/src/caffe/test/test_gradient_based_solver.cpp +++ b/src/caffe/test/test_gradient_based_solver.cpp @@ -180,6 +180,9 @@ class GradientBasedSolverTest : public MultiDeviceTest { proto << "momentum: " << momentum << " "; } MakeTempDir(&snapshot_prefix_); +#if defined(_MSC_VER) + std::replace(snapshot_prefix_.begin(), snapshot_prefix_.end(), '\\', '/'); +#endif proto << "snapshot_prefix: '" << snapshot_prefix_ << "/' "; if (snapshot) { proto << "snapshot: " << num_iters << " "; @@ -526,9 +529,8 @@ class GradientBasedSolverTest : public MultiDeviceTest { for (int i = 0; i < orig_params.size(); ++i) { param_copies[i].reset(new Blob()); const bool kReshape = true; - for (int copy_diff = false; copy_diff <= true; ++copy_diff) { - param_copies[i]->CopyFrom(*orig_params[i], copy_diff, kReshape); - } + param_copies[i]->CopyFrom(*orig_params[i], false/*copy data*/, kReshape); + param_copies[i]->CopyFrom(*orig_params[i], true/*copy diff*/, kReshape); } // Save the solver history @@ -538,9 +540,10 @@ class GradientBasedSolverTest : public MultiDeviceTest { for (int i = 0; i < orig_history.size(); ++i) { history_copies[i].reset(new Blob()); const bool kReshape = true; - for (int copy_diff = false; copy_diff <= true; ++copy_diff) { - history_copies[i]->CopyFrom(*orig_history[i], copy_diff, kReshape); - } + history_copies[i]->CopyFrom(*orig_history[i], + false/*copy data*/, kReshape); + history_copies[i]->CopyFrom(*orig_history[i], + true/*copy diff*/, kReshape); } // Run the solver for num_iters iterations and snapshot. diff --git a/src/caffe/test/test_lrn_layer.cpp b/src/caffe/test/test_lrn_layer.cpp index 4c97b1ae07b..23b52469a8f 100644 --- a/src/caffe/test/test_lrn_layer.cpp +++ b/src/caffe/test/test_lrn_layer.cpp @@ -279,11 +279,10 @@ class CuDNNLRNLayerTest : public GPUDeviceTest { vector*> blob_top_vec_; }; -template -void CuDNNLRNLayerTest::ReferenceLRNForward( - const Blob& blob_bottom, const LayerParameter& layer_param, - Blob* blob_top) { - typedef TypeParam Dtype; +template +void CuDNNLRNLayerTest::ReferenceLRNForward( + const Blob& blob_bottom, const LayerParameter& layer_param, + Blob* blob_top) { blob_top->Reshape(blob_bottom.num(), blob_bottom.channels(), blob_bottom.height(), blob_bottom.width()); Dtype* top_data = blob_top->mutable_cpu_data(); diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index 491a9bd03a6..c9da49e1e4b 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -1,6 +1,11 @@ #ifdef USE_LMDB #include "caffe/util/db_lmdb.hpp" +#if defined(_MSC_VER) +#include +#define mkdir(X, Y) _mkdir(X) +#endif + #include #include diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index 835d2d4e4ff..f679df8157a 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -1,4 +1,9 @@ #include + +#if defined(_MSC_VER) +#include +#endif + #include #include #include @@ -50,7 +55,11 @@ void WriteProtoToTextFile(const Message& proto, const char* filename) { } bool ReadProtoFromBinaryFile(const char* filename, Message* proto) { +#if defined (_MSC_VER) // for MSC compiler binary flag needs to be specified + int fd = open(filename, O_RDONLY | O_BINARY); +#else int fd = open(filename, O_RDONLY); +#endif CHECK_NE(fd, -1) << "File not found: " << filename; ZeroCopyInputStream* raw_input = new FileInputStream(fd); CodedInputStream* coded_input = new CodedInputStream(raw_input); diff --git a/src/caffe/util/signal_handler.cpp b/src/caffe/util/signal_handler.cpp index 5d764ec524f..6599db47159 100644 --- a/src/caffe/util/signal_handler.cpp +++ b/src/caffe/util/signal_handler.cpp @@ -13,9 +13,15 @@ namespace { void handle_signal(int signal) { switch (signal) { +#ifdef _MSC_VER + case SIGBREAK: // there is no SIGHUP in windows, take SIGBREAK instead. + got_sighup = true; + break; +#else case SIGHUP: got_sighup = true; break; +#endif case SIGINT: got_sigint = true; break; @@ -27,7 +33,14 @@ namespace { LOG(FATAL) << "Tried to hookup signal handlers more than once."; } already_hooked_up = true; - +#ifdef _MSC_VER + if (signal(SIGBREAK, handle_signal) == SIG_ERR) { + LOG(FATAL) << "Cannot install SIGBREAK handler."; + } + if (signal(SIGINT, handle_signal) == SIG_ERR) { + LOG(FATAL) << "Cannot install SIGINT handler."; + } +#else struct sigaction sa; // Setup the handler sa.sa_handler = &handle_signal; @@ -42,11 +55,20 @@ namespace { if (sigaction(SIGINT, &sa, NULL) == -1) { LOG(FATAL) << "Cannot install SIGINT handler."; } +#endif } // Set the signal handlers to the default. void UnhookHandler() { if (already_hooked_up) { +#ifdef _MSC_VER + if (signal(SIGBREAK, SIG_DFL) == SIG_ERR) { + LOG(FATAL) << "Cannot uninstall SIGBREAK handler."; + } + if (signal(SIGINT, SIG_DFL) == SIG_ERR) { + LOG(FATAL) << "Cannot uninstall SIGINT handler."; + } +#else struct sigaction sa; // Setup the sighub handler sa.sa_handler = SIG_DFL; @@ -61,7 +83,7 @@ namespace { if (sigaction(SIGINT, &sa, NULL) == -1) { LOG(FATAL) << "Cannot uninstall SIGINT handler."; } - +#endif already_hooked_up = false; } } diff --git a/src/gtest/CMakeLists.txt b/src/gtest/CMakeLists.txt index e98254af130..30a1553d588 100644 --- a/src/gtest/CMakeLists.txt +++ b/src/gtest/CMakeLists.txt @@ -1,7 +1,9 @@ add_library(gtest STATIC EXCLUDE_FROM_ALL gtest.h gtest-all.cpp) caffe_default_properties(gtest) target_include_directories(gtest PUBLIC ${Caffe_SRC_DIR}) -target_compile_definitions(gtest PUBLIC -DGTEST_USE_OWN_TR1_TUPLE) +if(NOT MSVC) + target_compile_definitions(gtest PUBLIC -DGTEST_USE_OWN_TR1_TUPLE) +endif() #add_library(gtest_main gtest_main.cc) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3789450555e..655cdc8d625 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,6 +15,10 @@ foreach(source ${srcs}) target_link_libraries(${name} ${Caffe_LINK}) caffe_default_properties(${name}) + if(MSVC AND COPY_PREREQUISITES) + caffe_copy_prerequisites(${name} USE_HARD_LINKS) + endif() + # set back RUNTIME_OUTPUT_DIRECTORY caffe_set_runtime_directory(${name} "${PROJECT_BINARY_DIR}/tools") caffe_set_solution_folder(${name} tools) @@ -22,9 +26,17 @@ foreach(source ${srcs}) # restore output name without suffix if(name MATCHES "caffe.bin") set_target_properties(${name} PROPERTIES OUTPUT_NAME caffe) + if(MSVC) + # the exectuable will have an import library with the same name as the caffe lib + # so change the import library to avoid name clashes + set_target_properties(${name} PROPERTIES IMPORT_SUFFIX ".bin.lib") + endif() endif() # Install install(TARGETS ${name} DESTINATION ${CMAKE_INSTALL_BINDIR}) + if(MSVC AND INSTALL_PREREQUISITES) + caffe_install_prerequisites(${name} DESTINATION ${CMAKE_INSTALL_BINDIR}) + endif() endforeach(source) diff --git a/tools/caffe.cpp b/tools/caffe.cpp index 3587d8aa1be..01b3dbd61e5 100644 --- a/tools/caffe.cpp +++ b/tools/caffe.cpp @@ -174,6 +174,7 @@ caffe::SolverAction::Enum GetRequestedAction( return caffe::SolverAction::NONE; } LOG(FATAL) << "Invalid signal effect \""<< flag_value << "\" was specified"; + return caffe::SolverAction::NONE; } // Train / Finetune a model.