Speed up build system

.gitignore

@@ -91,3 +91,6 @@ node_modules/
 # generated by CMake + clang - Symlink debug/compile_commands.json (or release)
 # to the root of the project for most tooling to work. This ignores the symlink.
 /compile_commands.json
+
+# Visual Studio Code files
+/.vscode/

CMakeLists.txt

@@ -736,29 +736,47 @@ macro (make_copy_target NAME)
   CMAKE_PARSE_ARGUMENTS(copy "" "" "${multi_value_args}" ${ARGN})
   ADD_CUSTOM_TARGET(${NAME} ALL)
   set(all_target_locations)
-  foreach(binary ${copy_FILES})
-          # Switched away from cmake -E commands because they don't work right
-          # on Windows. Also Windows doesn't overwrite when copying by default
-          if(WIN32)
+  set(RSYNC_SOURCES ${copy_FILES} ${copy_DIRECTORIES})
+  find_program(RSYNC rsync)
+  if("${RSYNC}" STREQUAL "")
+    # No rsync present; copy the old fashioned way with multiple cp commands
+    foreach(binary ${copy_FILES})
+            # Switched away from cmake -E commands because they don't work right
+            # on Windows. Also Windows doesn't overwrite when copying by default
+            if(WIN32)
+              add_custom_command(TARGET ${NAME} POST_BUILD
+                COMMENT "remove old ${binary}"
+                COMMAND rm -f ${CMAKE_CURRENT_BINARY_DIR}/`basename ${binary}` )
+            endif()
             add_custom_command(TARGET ${NAME} POST_BUILD
-              COMMENT "remove old ${binary}"
-              COMMAND rm -f ${CMAKE_CURRENT_BINARY_DIR}/`basename ${binary}` )
-          endif()
-          add_custom_command(TARGET ${NAME} POST_BUILD
-                  COMMENT "copy ${binary}"
-                  COMMAND cp ${binary} ${CMAKE_CURRENT_BINARY_DIR})
+                    COMMENT "copy ${binary}"
+                    COMMAND cp ${binary} ${CMAKE_CURRENT_BINARY_DIR})
+    endforeach()
+
+    foreach(binary ${copy_DIRECTORIES})
+            # Switched away from cmake -E commands because they don't work right
+            # on Windows. Also Windows doesn't overwrite when copying by default
+            add_custom_command(TARGET ${NAME} POST_BUILD
+                    COMMENT "remove old ${binary}"
+                    COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/`basename ${binary}` )
+            add_custom_command(TARGET ${NAME} POST_BUILD
+                    COMMENT "copy ${binary}"
+                    COMMAND cp -r ${binary} ${CMAKE_CURRENT_BINARY_DIR})
+    endforeach()
+  else()
+    if((NOT "${copy_FILES}" STREQUAL "") OR (NOT "${copy_DIRECTORIES}" STREQUAL ""))
+      add_custom_command(TARGET ${NAME} POST_BUILD
+        COMMENT "copying ${NAME}"
+        COMMAND ${RSYNC} -a ${RSYNC_SOURCES} ${CMAKE_CURRENT_BINARY_DIR}
+      )
+    endif()
+  endif()
+
+  # Make sure the files and directories get included in all_target_locations
+  foreach(binary ${copy_FILES})
           list(APPEND all_target_locations ${binary})
   endforeach()
-
   foreach(binary ${copy_DIRECTORIES})
-          # Switched away from cmake -E commands because they don't work right
-          # on Windows. Also Windows doesn't overwrite when copying by default
-          add_custom_command(TARGET ${NAME} POST_BUILD
-                  COMMENT "remove old ${binary}"
-                  COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/`basename ${binary}` )
-          add_custom_command(TARGET ${NAME} POST_BUILD
-                  COMMENT "copy ${binary}"
-                  COMMAND cp -r ${binary} ${CMAKE_CURRENT_BINARY_DIR})
           list(APPEND all_target_locations ${binary})
   endforeach()
 
@@ -776,15 +794,17 @@ macro (make_copy_target NAME)
                   foreach(location ${CUSTOM_LOCATION})
                           add_custom_command(TARGET ${NAME} POST_BUILD
                                   COMMENT "copy ${location}"
+                                  DEPENDS ${target}
                                   COMMAND ${CMAKE_COMMAND} -E
-                                  copy ${location} ${CMAKE_CURRENT_BINARY_DIR})
+                                  copy_if_different ${location} ${CMAKE_CURRENT_BINARY_DIR})
                           list(APPEND all_target_locations ${location})
                   endforeach()
           else()
                   add_custom_command(TARGET ${NAME} POST_BUILD
                           COMMENT "copying target of ${target}"
+                          DEPENDS ${target}
                           COMMAND ${CMAKE_COMMAND} -E
-                          copy $<TARGET_FILE:${target}> ${CMAKE_CURRENT_BINARY_DIR})
+                          copy_if_different $<TARGET_FILE:${target}> ${CMAKE_CURRENT_BINARY_DIR})
                   list(APPEND all_target_locations $<TARGET_FILE:${target}>)
           endif()
   endforeach()

cmake/SharedLibraryFromStatic.cmake

@@ -72,6 +72,16 @@ function(make_shared_library_from_static NAME)
 
         set_target_properties(${NAME}_collect PROPERTIES RULE_LAUNCH_LINK "env WORKINGDIR=${CMAKE_CURRENT_BINARY_DIR}/${NAME}_collect_dir BUILD_PREFIX=${CMAKE_BINARY_DIR} bash ${CMAKE_SOURCE_DIR}/scripts/dump_library_list.sh")
 
+        add_custom_command(
+          OUTPUT
+          ${CMAKE_CURRENT_BINARY_DIR}/${NAME}_collect_dir/collect_archives.txt
+          ${CMAKE_CURRENT_BINARY_DIR}/${NAME}_collect_dir/collect_libs.txt
+          COMMAND true
+          COMMENT "Dumping library/archive list for ${NAME}"
+          DEPENDS ${NAME}_collect
+          VERBATIM
+        )
+
         if (${args_DIRECT})
                 # add a custom command after link to override the collected list
                 add_custom_command(TARGET ${NAME}_collect
@@ -93,10 +103,10 @@ function(make_shared_library_from_static NAME)
                         REQUIRES ${args_REQUIRES})
         endif()
         if(NOT ${args_EXPORT_MAP} STREQUAL "")
-                set_target_properties(${NAME} PROPERTIES RULE_LAUNCH_LINK "env WORKINGDIR=${NAME}_collect_dir BUILD_PREFIX=${CMAKE_BINARY_DIR} EXPORT_MAP=${args_EXPORT_MAP} bash ${CMAKE_SOURCE_DIR}/scripts/collect_archive.sh")
-                set_property(TARGET ${NAME} APPEND PROPERTY LINK_DEPENDS "${args_EXPORT_MAP}")
+          set_target_properties(${NAME} PROPERTIES RULE_LAUNCH_LINK "env WORKINGDIR=${CMAKE_CURRENT_BINARY_DIR}/${NAME}_collect_dir BUILD_PREFIX=${CMAKE_BINARY_DIR} EXPORT_MAP=${args_EXPORT_MAP} bash ${CMAKE_SOURCE_DIR}/scripts/collect_archive.sh")
+          set_property(TARGET ${NAME} APPEND PROPERTY LINK_DEPENDS "${args_EXPORT_MAP}")
         else()
-                set_target_properties(${NAME} PROPERTIES RULE_LAUNCH_LINK "env WORKINGDIR=${NAME}_collect_dir BUILD_PREFIX=${CMAKE_BINARY_DIR} bash ${CMAKE_SOURCE_DIR}/scripts/collect_archive.sh")
+          set_target_properties(${NAME} PROPERTIES RULE_LAUNCH_LINK "env WORKINGDIR=${CMAKE_CURRENT_BINARY_DIR}/${NAME}_collect_dir BUILD_PREFIX=${CMAKE_BINARY_DIR} bash ${CMAKE_SOURCE_DIR}/scripts/collect_archive.sh")
         endif()
         add_dependencies(${NAME} ${NAME}_collect)
 

cmake/UseCython.cmake

@@ -214,9 +214,7 @@ function( compile_pyx _name generated_file )
   endforeach()
 
   # Determining generated file name.
-  set( _generated_file "${CMAKE_CURRENT_BINARY_DIR}/${_name}.${extension}"
-                       "${CMAKE_CURRENT_BINARY_DIR}/${_name}.h"
-                       "${CMAKE_CURRENT_BINARY_DIR}/${_name}_api.h")
+  set( _generated_file "${CMAKE_CURRENT_BINARY_DIR}/${_name}.${extension}")
   set( _generated_target_file "${CMAKE_CURRENT_BINARY_DIR}/${_name}.${extension}")
   set_source_files_properties( ${_generated_file} PROPERTIES GENERATED TRUE )
   set_source_files_properties( ${_generated_file} PROPERTIES COMPILE_FLAGS "-Wno-cpp -Wno-unused-function" )

deps/cmake/ExternalProjectBoost.cmake

@@ -80,6 +80,18 @@ ExternalProject_Add(ex_boost
   ${EXTRA_CONFIGURE_COMMANDS}
   BUILD_COMMAND ${BOOST_BUILD_COMMAND}
   INSTALL_COMMAND ${BOOST_INSTALL_COMMAND}
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_chrono.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_context.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_coroutine.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_date_time.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_iostreams.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_filesystem.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_program_options.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_random.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_regex.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_system.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_unit_test_framework.a
+                   ${CMAKE_SOURCE_DIR}/deps/local/lib/libboost_thread.a
   )
 
 set(Boost_LIBRARIES

deps/cmake/ExternalProjectLibCurl.cmake

@@ -11,6 +11,7 @@ ExternalProject_Add(ex_libcurl
   URL ${CMAKE_SOURCE_DIR}/deps/src/curl-7.33.0
   INSTALL_DIR ${CMAKE_SOURCE_DIR}/deps/local
   CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CPPFLAGS=-fPIC <SOURCE_DIR>/configure --prefix=<INSTALL_DIR> --without-winidn --without-libidn --without-nghttp2 --without-ca-bundle --without-polarssl --without-cyassl --without-nss --disable-crypto-auth --enable-shared=no --enable-static=yes --disable-ldap --without-librtmp --without-zlib --libdir=<INSTALL_DIR>/lib ${EXTRA_CONFIGURE_FLAGS}
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libcurl.a
   )
 
 add_library(libcurla STATIC IMPORTED)

deps/cmake/ExternalProjectLibJpeg.cmake

@@ -9,6 +9,7 @@ ExternalProject_Add(ex_libjpeg
   INSTALL_DIR ${CMAKE_SOURCE_DIR}/deps/local
   CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CPPFLAGS=-fPIC <SOURCE_DIR>/configure --enable-shared=no --prefix=<INSTALL_DIR> ${EXTRA_CONFIGURE_FLAGS}
   INSTALL_COMMAND make install
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libjpeg.a
   BUILD_IN_SOURCE 1)
 
 add_library(libjpega STATIC IMPORTED)

deps/cmake/ExternalProjectLibPNG.cmake

@@ -10,6 +10,7 @@ ExternalProject_Add(ex_libpng
   INSTALL_DIR ${CMAKE_SOURCE_DIR}/deps/local
   CONFIGURE_COMMAND env CFLAGS=${CFLAGS} CPPFLAGS=${CFLAGS} <SOURCE_DIR>/configure --enable-shared=no --prefix=<INSTALL_DIR> ${EXTRA_CONFIGURE_FLAGS}
   INSTALL_COMMAND make install
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libpng.a
   BUILD_IN_SOURCE 1)
 
 add_dependencies(ex_libpng libza)

deps/cmake/ExternalProjectLibXML2.cmake

@@ -9,6 +9,7 @@ ExternalProject_Add(ex_libxml2
   INSTALL_DIR ${CMAKE_SOURCE_DIR}/deps/local
   CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CPPFLAGS=-fPIC <SOURCE_DIR>/configure --prefix=<INSTALL_DIR> --enable-shared=no --enable-static=yes --without-lzma --libdir=<INSTALL_DIR>/lib --with-python=./ ${EXTRA_CONFIGURE_FLAGS}
   BUILD_COMMAND cp <SOURCE_DIR>/testchar.c <SOURCE_DIR>/testapi.c && make
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libxml2.a
   )
 # the with-python=./ prevents it from trying to build/install some python stuff
 # which is poorly installed (always ways to stick it in a system directory)

deps/cmake/ExternalProjectLibz.cmake

@@ -17,6 +17,7 @@ ExternalProject_Add(ex_libz
   CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CFLAGS=-fPIC ./configure --static --64 --prefix=<INSTALL_DIR>
   BUILD_IN_SOURCE 1
   BUILD_COMMAND CC=${CMAKE_C_COMPILER} CFLAGS=-fPIC make install
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libz.a
   INSTALL_COMMAND "" )
 endif()
 add_library(libza STATIC IMPORTED)

deps/cmake/ExternalProjectNanoMSG.cmake

@@ -5,6 +5,7 @@ ExternalProject_Add(ex_libnanomsg
   INSTALL_DIR ${CMAKE_SOURCE_DIR}/deps/local
   CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CPPFLAGS=-fPIC ${CMAKE_COMMAND} -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> -DCMAKE_INSTALL_LIBDIR=lib -DNN_STATIC_LIB=ON .
   INSTALL_COMMAND make install
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libnanomsg.a
   BUILD_IN_SOURCE 1)
 
 add_library(libnanomsga STATIC IMPORTED)

deps/cmake/ExternalProjectOpenSSL.cmake

@@ -10,6 +10,7 @@ ExternalProject_Add(ex_libssl
   CONFIGURE_COMMAND ./Configure darwin64-x86_64-cc no-idea no-mdc2 no-rc5 -fPIC --prefix=<INSTALL_DIR>
   BUILD_COMMAND make -j1
   INSTALL_COMMAND make -j1 install && cp ./libcrypto.a <INSTALL_DIR>/ssl && cp ./libssl.a <INSTALL_DIR>/ssl
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libssl.a ${CMAKE_SOURCE_DIR}/deps/local/lib/libcrypto.a
   )
 elseif(WIN32)
 ExternalProject_Add(ex_libssl
@@ -30,6 +31,7 @@ ExternalProject_Add(ex_libssl
   CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} ./config no-idea no-mdc2 no-rc5 -fPIC --prefix=<INSTALL_DIR>
   BUILD_COMMAND make -j1
   INSTALL_COMMAND make -j1 install_sw
+  BUILD_BYPRODUCTS ${CMAKE_SOURCE_DIR}/deps/local/lib/libssl.a ${CMAKE_SOURCE_DIR}/deps/local/lib/libcrypto.a
   )
 endif()
 

src/unity/lib/toolkit_class_macros.hpp

@@ -74,9 +74,7 @@
      const char* file = __FILE__; \
      file = ((strrchr(file, '/') ? : file- 1) + 1); \
      return std::string(file) + ":" + \
-         std::to_string(__LINE__) +  " " + \
-         std::string(__DATE__) + " " + \
-         std::string(__TIME__); \
+         std::to_string(__LINE__); \
    } \
    virtual inline void perform_registration() { \
     if (registered) return;

src/unity/python/CMakeLists.txt

@@ -12,23 +12,24 @@ add_subdirectory(turicreate)
 # Copy all the open source python stuff over first
 
 # Overwrite with the proprietary stuff
-ADD_CUSTOM_TARGET (python_source ALL)
-ADD_CUSTOM_COMMAND(
-  COMMENT "build prop python source"
-  COMMAND
-  ${CMAKE_COMMAND} -E
-  copy_directory ${CMAKE_CURRENT_SOURCE_DIR}
-  ${CMAKE_CURRENT_BINARY_DIR}
 
-  TARGET python_source
+make_copy_target(python_source
+  FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt
+    ${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in
+    ${CMAKE_CURRENT_SOURCE_DIR}/README.rst
+    ${CMAKE_CURRENT_SOURCE_DIR}/dummy.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg
+    ${CMAKE_CURRENT_SOURCE_DIR}/setup.py
+  DIRECTORIES
+    ${CMAKE_CURRENT_SOURCE_DIR}/turicreate
 )
 add_dependencies(python_source numpy_integration)
 
-
-ADD_CUSTOM_TARGET (clean_python)
+add_custom_target (clean_python)
 add_dependencies(clean_python clean_turicreate_binaries clean_cython)
 
-ADD_CUSTOM_COMMAND(
+add_custom_command(
   COMMENT "make clean python directory"
   COMMAND
   ${CMAKE_COMMAND} -P

src/unity/python/turicreate/CMakeLists.txt

@@ -25,7 +25,7 @@ foreach(loop_var ${EXTENSIONS_LIST})
   list(APPEND INSTALLATION_EXTENSIONS ${loop_var})
 endforeach()
 
-make_copy_target(release_binaries ALL
+make_copy_target(release_binaries
         TARGETS
         ${UNITY_ALL_LIBRARIES}
         ${INSTALLATION_BINARY_TARGETS}
@@ -49,25 +49,29 @@ set_property(DIRECTORY PROPERTY INSTALLATION_EXTENSIONS "${INSTALLATION_EXTENSIO
 set_property(DIRECTORY PROPERTY INSTALLATION_BINARY_TARGETS "${INSTALLATION_BINARY_TARGETS}")
 set_property(DIRECTORY PROPERTY INSTALLATION_BINARY_FILES "${INSTALLATION_BINARY_FILES}")
 
-add_custom_target(
-  visualization_vega ALL
-  DEPENDS visualization_client
-  COMMAND cp "${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vega_3.2.1.js" "${CMAKE_CURRENT_BINARY_DIR}"
-  COMMAND cp "${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vg2png" "${CMAKE_CURRENT_BINARY_DIR}"
-  COMMAND cp "${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vg2svg" "${CMAKE_CURRENT_BINARY_DIR}"
-)
-
 if (APPLE)
-  add_custom_target(
-    visualization_client_app ALL
-    DEPENDS visualization_client
-    COMMAND cp -a "${CMAKE_BINARY_DIR}/src/visualization/${CMAKE_BUILD_TYPE}/Turi Create Visualization.app" ${CMAKE_CURRENT_BINARY_DIR}
+  make_copy_target(
+    visualization_client_app
+    FILES
+    ${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vega_3.2.1.js
+    ${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vg2png
+    ${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vg2svg
+    DIRECTORIES
+    "${CMAKE_BINARY_DIR}/src/visualization/${CMAKE_BUILD_TYPE}/Turi Create Visualization.app"
   )
+  add_dependencies(visualization_client_app visualization_client)
 elseif(LINUX)
   add_custom_target(
     visualization_client_app ALL
     DEPENDS visualization_client
     COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/Turi Create Visualization"
     COMMAND cp -a "${CMAKE_BINARY_DIR}/src/visualization/TcViz/*" "${CMAKE_CURRENT_BINARY_DIR}/Turi Create Visualization"
   )
+  add_custom_target(
+    visualization_vega ALL
+    DEPENDS visualization_client
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vega_3.2.1.js" "${CMAKE_CURRENT_BINARY_DIR}"
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vg2png" "${CMAKE_CURRENT_BINARY_DIR}"
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/src/unity/python/turicreate/visualization/vg2svg" "${CMAKE_CURRENT_BINARY_DIR}"
+  )
 endif()

src/unity/python/turicreate/cython/CMakeLists.txt

@@ -11,7 +11,7 @@ include_directories(${NUMPY_INCLUDE_DIRS})
 function(cython_add_turicreate_module _module_name _sources)
   set_source_files_properties( ${_sources} PROPERTIES CYTHON_IS_CXX TRUE)
   cython_add_module( ${_module_name} ${_sources} )
-  set_property(TARGET ${_module_name} APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-rpath,$ORIGIN -Wl,-rpath,$ORIGIN/.. -Wl,-rpath,${CMAKE_CURRENT_BINARY_DIR} ")
+  set_property(TARGET ${_module_name} APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-rpath,\\$ORIGIN -Wl,-rpath,\\$ORIGIN/.. -Wl,-rpath,${CMAKE_CURRENT_BINARY_DIR} ")
   target_link_libraries(${_module_name} boost nanomsg ${UNITY_EXTENSION_LIBRARY})
   set_property(TARGET ${_module_name} APPEND_STRING PROPERTY COMPILE_FLAGS " -include ${CMAKE_CURRENT_SOURCE_DIR}/cython_cpp_error_handler.hpp ")
   add_dependencies(${_module_name} release_binaries)