CMake: Add option COMPILE_D_MODULES_SEPARATELY

And enable it for 1st Shippable build, decreasing the build time by a
very rough 3 minutes.

CMakeLists.txt

@@ -14,6 +14,7 @@ include(CheckLibraryExists)
 include(CheckCXXCompilerFlag)
 include(CheckDSourceCompiles)
 include(CheckLinkFlag)
+include(BuildDExecutable)
 
 # The script currently only supports the DMD-style commandline interface
 if (NOT D_COMPILER_DMD_COMPAT)
@@ -91,6 +92,8 @@ set(CONF_INST_DIR ${SYSCONF_INSTALL_DIR} CACHE PATH "Directory ldc.conf is insta
 # Note: LIB_SUFFIX should perhaps be renamed to LDC_LIBDIR_SUFFIX.
 set(LIB_SUFFIX "" CACHE STRING "Appended to the library installation directory. Set to '64' to install libraries into ${PREFIX}/lib64.")
 
+set(COMPILE_D_MODULES_SEPARATELY OFF CACHE BOOL "Compile each D module separately (instead of all at once). Useful for many CPU cores and/or iterative development; generated executables will be somewhat slower.")
+
 # The following flag is currently not well tested, expect the build to fail.
 option(GENERATE_OFFTI "generate complete ClassInfo.offTi arrays")
 mark_as_advanced(GENERATE_OFFTI)
@@ -640,42 +643,6 @@ else()
     endif()
 endif()
 
-function(build_d_executable output_exe compiler_args linker_args compile_deps link_deps)
-    # Compile all D modules to a single object.
-    set(object_file ${output_exe}${CMAKE_CXX_OUTPUT_EXTENSION})
-    set(dflags "${D_COMPILER_FLAGS} ${DDMD_DFLAGS}")
-    if(UNIX)
-      separate_arguments(dflags UNIX_COMMAND "${dflags}")
-    else()
-      separate_arguments(dflags WINDOWS_COMMAND "${dflags}")
-    endif()
-    add_custom_command(
-        OUTPUT ${object_file}
-        COMMAND ${D_COMPILER} -c ${dflags} -of${object_file} ${compiler_args}
-        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-        DEPENDS ${compile_deps}
-    )
-    # Link to an executable.
-    if(LDC_LINK_MANUALLY)
-        add_custom_command(
-            OUTPUT ${output_exe}
-            COMMAND ${CMAKE_CXX_COMPILER} -o ${output_exe} ${object_file} ${linker_args} ${LDC_LINKERFLAG_LIST}
-            DEPENDS ${object_file} ${link_deps}
-        )
-    else()
-        set(translated_linker_flags "")
-        foreach(f ${linker_args})
-            list(APPEND translated_linker_flags "-L${f}")
-        endforeach()
-        add_custom_command(
-            OUTPUT ${output_exe}
-            COMMAND ${D_COMPILER} ${dflags} ${DDMD_LFLAGS} -of${output_exe} ${object_file} ${translated_linker_flags} ${LDC_TRANSLATED_LINKER_FLAGS}
-            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            DEPENDS ${object_file} ${link_deps}
-        )
-    endif()
-endfunction()
-
 # CONFIG generator expressions need to be repeated due to https://cmake.org/Bug/view.php?id=14353
 if(MSVC_IDE)
     separate_arguments(LDC_FLAG_LIST WINDOWS_COMMAND "${LDC_TRANSLATED_LINKER_FLAGS} ${D_COMPILER_FLAGS} ${DDMD_DFLAGS} ${DDMD_LFLAGS}")
@@ -689,8 +656,9 @@ else()
     build_d_executable(
         "${LDC_EXE_FULL}"
         "${LDC_D_SOURCE_FILES}"
+        ""
         "$<TARGET_LINKER_FILE:${LDC_LIB}>"
-        "${LDC_D_SOURCE_FILES};${FE_RES}"
+        "${FE_RES}"
         "${LDC_LIB}"
     )
 endif()
@@ -725,12 +693,13 @@ set_target_properties(
     ARCHIVE_OUTPUT_NAME ldmd
     LIBRARY_OUTPUT_NAME ldmd
 )
-set(LDMD_D_SOURCE_FILES dmd/root/man.d driver/ldmd.d)
+set(LDMD_D_SOURCE_FILES ${PROJECT_SOURCE_DIR}/dmd/root/man.d ${PROJECT_SOURCE_DIR}/driver/ldmd.d)
 build_d_executable(
     "${LDMD_EXE_FULL}"
     "${LDMD_D_SOURCE_FILES}"
+    ""
     "$<TARGET_LINKER_FILE:LDMD_CXX_LIB>"
-    "${LDMD_D_SOURCE_FILES}"
+    ""
     "LDMD_CXX_LIB"
 )
 
@@ -867,9 +836,10 @@ enable_testing()
 set(LDC_UNITTEST_EXE_FULL ${PROJECT_BINARY_DIR}/bin/${LDC_EXE_NAME}-unittest${CMAKE_EXECUTABLE_SUFFIX})
 build_d_executable(
     "${LDC_UNITTEST_EXE_FULL}"
-    "-unittest;${LDC_D_SOURCE_FILES}"
-    "$<TARGET_LINKER_FILE:${LDC_LIB}>"
     "${LDC_D_SOURCE_FILES}"
+    "-unittest"
+    "$<TARGET_LINKER_FILE:${LDC_LIB}>"
+    ""
     "${LDC_LIB}"
 )
 add_custom_target(ldc2-unittest DEPENDS ${LDC_UNITTEST_EXE_FULL})
@@ -894,6 +864,7 @@ build_d_executable(
     "${LDC_BUILD_RUNTIME_EXE_FULL}"
     "${PROJECT_BINARY_DIR}/ldc-build-runtime.d"
     ""
+    ""
     "${PROJECT_SOURCE_DIR}/runtime/ldc-build-runtime.d.in"
     ""
 )

cmake/Modules/BuildDExecutable.cmake

@@ -0,0 +1,76 @@
+# Depends on these global variables:
+# - COMPILE_D_MODULES_SEPARATELY
+# - D_COMPILER
+# - D_COMPILER_FLAGS
+# - DDMD_DFLAGS
+# - DDMD_LFLAGS
+# - LDC_LINK_MANUALLY
+# - LDC_LINKERFLAG_LIST
+# - LDC_TRANSLATED_LINKER_FLAGS
+function(build_d_executable output_exe d_src_files compiler_args linker_args extra_compile_deps link_deps)
+    set(dflags "${D_COMPILER_FLAGS} ${DDMD_DFLAGS}")
+    if(UNIX)
+      separate_arguments(dflags UNIX_COMMAND "${dflags}")
+    else()
+      separate_arguments(dflags WINDOWS_COMMAND "${dflags}")
+    endif()
+
+    get_filename_component(exe_basename ${output_exe} NAME_WE) # no path, no extension
+
+    set(object_files)
+    if(NOT COMPILE_D_MODULES_SEPARATELY)
+        # Compile all D modules to a single object.
+        set(object_file ${PROJECT_BINARY_DIR}/obj/${exe_basename}${CMAKE_CXX_OUTPUT_EXTENSION})
+        add_custom_command(
+            OUTPUT ${object_file}
+            COMMAND ${D_COMPILER} -c ${dflags} -of${object_file} ${compiler_args} ${d_src_files}
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            DEPENDS ${d_src_files} ${extra_compile_deps}
+        )
+        set(object_files ${object_file})
+    else()
+        # Compile each D module separately.
+        foreach(f ${d_src_files})
+            file(RELATIVE_PATH object_file ${PROJECT_SOURCE_DIR} ${f})       # make path relative to PROJECT_SOURCE_DIR
+            string(REGEX REPLACE "[/\\\\]" "." object_file "${object_file}") # replace path separators with '.'
+            string(REGEX REPLACE "^\\.+" "" object_file "${object_file}")    # strip leading dots (e.g., from original '../dir/file.d' => '...dir.file.d' => 'dir.file.d')
+            set(object_file ${PROJECT_BINARY_DIR}/obj/${exe_basename}/${object_file}${CMAKE_CXX_OUTPUT_EXTENSION})
+            add_custom_command(
+                OUTPUT ${object_file}
+                COMMAND ${D_COMPILER} -c ${dflags} -of${object_file} ${compiler_args} ${f}
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                DEPENDS ${f} ${extra_compile_deps}
+            )
+            list(APPEND object_files ${object_file})
+        endforeach()
+    endif()
+
+    # Use a response file on Windows when compiling separately, in order not to
+    # exceed the max command-line length.
+    set(objects_args "${object_files}")
+    if(WIN32 AND COMPILE_D_MODULES_SEPARATELY)
+        string(REPLACE ";" " " objects_args "${object_files}")
+        file(WRITE ${output_exe}.rsp ${objects_args})
+        set(objects_args "@${output_exe}.rsp")
+    endif()
+
+    # Link to an executable.
+    if(LDC_LINK_MANUALLY)
+        add_custom_command(
+            OUTPUT ${output_exe}
+            COMMAND ${CMAKE_CXX_COMPILER} -o ${output_exe} ${objects_args} ${linker_args} ${LDC_LINKERFLAG_LIST}
+            DEPENDS ${object_files} ${link_deps}
+        )
+    else()
+        set(translated_linker_args "")
+        foreach(f ${linker_args})
+            list(APPEND translated_linker_args "-L${f}")
+        endforeach()
+        add_custom_command(
+            OUTPUT ${output_exe}
+            COMMAND ${D_COMPILER} ${dflags} ${DDMD_LFLAGS} -of${output_exe} ${objects_args} ${translated_linker_args} ${LDC_TRANSLATED_LINKER_FLAGS}
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            DEPENDS ${object_files} ${link_deps}
+        )
+    endif()
+endfunction()

runtime/CMakeLists.txt

@@ -40,7 +40,7 @@ set(BUILD_SHARED_LIBS     AUTO                                CACHE STRING "Whet
 set(D_FLAGS               -w                                  CACHE STRING "Runtime D compiler flags, separated by ';'")
 set(D_FLAGS_DEBUG         -g;-link-defaultlib-debug           CACHE STRING "Runtime D compiler flags (debug libraries), separated by ';'")
 set(D_FLAGS_RELEASE       -O3;-release                        CACHE STRING "Runtime D compiler flags (release libraries), separated by ';'")
-set(COMPILE_ALL_D_FILES_AT_ONCE ON                            CACHE BOOL   "Compile all D files for a lib in a single command line instead of separately")
+set(COMPILE_ALL_D_FILES_AT_ONCE ON                            CACHE BOOL   "Compile all D files for the runtime libs in a single command line instead of separately. Disabling this is useful for many CPU cores and/or iterative development.")
 set(RT_ARCHIVE_WITH_LDC   ON                                  CACHE STRING "Whether to archive the static runtime libs via LDC instead of CMake archiver")
 set(RT_CFLAGS             ""                                  CACHE STRING "Runtime extra C compiler flags, separated by ' '")
 set(LD_FLAGS              ""                                  CACHE STRING "Runtime extra C linker flags, separated by ' '")

shippable.yml

@@ -62,6 +62,7 @@ build:
         -DLLVM_ROOT_DIR=$PWD/../llvm \
         -DCMAKE_INSTALL_PREFIX=$PWD/../ldc-bootstrap \
         -DD_COMPILER=$PWD/../ldc-ltsmaster/bin/ldmd2 \
+        -DCOMPILE_D_MODULES_SEPARATELY=ON \
         ..
     - ninja -j32 install
     - cd ..