8210459: Add support for generating compile_commands.json

Reviewed-by: erikj, ihse

make/CompileCommands.gmk

@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+default: all
+
+include $(SPEC)
+include MakeBase.gmk
+
+# When FIXPATH is set, let it process the file to make sure all paths are usable
+# by system native tools. The FIXPATH tool assumes arguments preceeded by an @
+# character points to a text file containing further arguments (similar to a
+# linker). It replaces any such arguments with a different temporary filename,
+# whose contents has been processed to make any paths native. To obtain a
+# properly processed compile_commands.json, FIXPATH is then made to invoke an
+# AWK script with the unprocessed json file as the only argument, prepended with
+# an @ character. The AWK script simply copies the contents of this processed
+# file.
+#
+# The sed command encloses the fragments inside brackets and removes the final
+# trailing comma.
+$(OUTPUTDIR)/compile_commands.json: $(wildcard $(MAKESUPPORT_OUTPUTDIR)/compile-commands/*.json)
+	$(call LogWarn, Updating compile_commands.json)
+	$(RM) $@
+	$(FIND) $(MAKESUPPORT_OUTPUTDIR)/compile-commands/ -name \*.json | \
+	    $(SORT) | $(XARGS) $(CAT) >> $@.tmp
+	$(if $(FIXPATH),$(FIXPATH) $(AWK) 'BEGIN { \
+	    tmpfile = substr(ARGV[2],2); \
+	    cmd = "$(CP) " "\047" tmpfile "\047" " $@.tmp"; \
+	    system(cmd); \
+	}' -- @$@.tmp)
+	$(SED) -e '1s/^/[\$(NEWLINE)/' -e '$(DOLLAR)s/,\s\{0,\}$(DOLLAR)/\$(NEWLINE)]/' $@.tmp > $@
+	$(RM) $@.tmp
+
+TARGETS += $(OUTPUTDIR)/compile_commands.json
+
+all: $(TARGETS)
+
+.PHONY: all

make/Main.gmk

@@ -266,6 +266,31 @@ hotspot-ide-project:
 ALL_TARGETS += $(HOTSPOT_VARIANT_TARGETS) $(HOTSPOT_VARIANT_GENSRC_TARGETS) \
     $(HOTSPOT_VARIANT_LIBS_TARGETS) hotspot-ide-project
 
+################################################################################
+# Generate libs and launcher targets for creating compile_commands.json fragments
+define DeclareCompileCommandsRecipe
+  $1-compile-commands:
+	$$(call LogInfo, Generating compile_commands.json fragments for $1)
+	+($(CD) $(TOPDIR)/make && $(MAKE) $(MAKE_ARGS) -f Main.gmk $1-only \
+	    GENERATE_COMPILE_COMMANDS_ONLY=true)
+
+  COMPILE_COMMANDS_TARGETS_$2 += $1-compile-commands
+endef
+
+$(foreach t, $(HOTSPOT_VARIANT_LIBS_TARGETS), \
+  $(eval $(call DeclareCompileCommandsRecipe,$t,HOTSPOT)) \
+)
+
+$(foreach t, $(LIBS_TARGETS) $(LAUNCHER_TARGETS), \
+  $(eval $(call DeclareCompileCommandsRecipe,$t,JDK)) \
+)
+
+compile-commands compile-commands-hotspot:
+	+($(CD) $(TOPDIR)/make && $(MAKE) $(MAKE_ARGS) -f CompileCommands.gmk)
+
+ALL_TARGETS += $(COMPILE_COMMANDS_TARGETS_HOTSPOT) $(COMPILE_COMMANDS_TARGETS_JDK)
+ALL_TARGETS += compile-commands compile-commands-hotspot
+
 ################################################################################
 # Build demos targets
 
@@ -563,8 +588,12 @@ test-jdk-jtreg-native:
 test-make:
 	($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f TestMake.gmk $(TEST_TARGET))
 
+test-compile-commands:
+	($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f TestMake.gmk test-compile-commands)
+
 ALL_TARGETS += test test-hotspot-jtreg test-hotspot-jtreg-native \
-    test-hotspot-internal test-hotspot-gtest test-jdk-jtreg-native test-make
+    test-hotspot-internal test-hotspot-gtest test-jdk-jtreg-native test-make \
+    test-compile-commands
 
 ################################################################################
 # Bundles
@@ -744,6 +773,20 @@ else
   $(foreach m, $(ALL_MODULES), $(eval $m-jmod: $($(m)_JMOD_DEPS)))
   $(foreach m, $(INTERIM_IMAGE_MODULES), $(eval $m-interim-jmod: $($(m)_JMOD_DEPS)))
 
+  # Setup the minimal set of generated native source dependencies for hotspot
+  $(foreach v, $(JVM_VARIANTS), \
+    $(eval hotspot-$v-libs-compile-commands: hotspot-$v-gensrc) \
+    $(foreach m, $(filter java.desktop jdk.hotspot.agent, $(GENSRC_MODULES)), \
+      $(eval hotspot-$v-libs-compile-commands: $m-gensrc)) \
+  )
+
+  # For the full JDK compile commands, create all possible generated sources
+  $(foreach m, $(GENSRC_MODULES), $(eval $m-libs-compile-commands: $m-gensrc))
+  $(foreach m, $(filter $(JAVA_MODULES), $(LIBS_MODULES)), $(eval $m-libs-compile-commands: $m-java))
+
+  compile-commands-hotspot: $(COMPILE_COMMANDS_TARGETS_HOTSPOT)
+  compile-commands: $(COMPILE_COMMANDS_TARGETS_HOTSPOT) $(COMPILE_COMMANDS_TARGETS_JDK)
+
   # Jmods cannot be created until we have the jmod tool ready to run. During
   # a normal build we run it from the exploded image, but when cross compiling
   # it's run from the buildjdk, which is either created at build time or user
@@ -858,6 +901,8 @@ else
 
   test-make: clean-test-make
 
+  test-compile-commands: compile-commands
+
   build-test-lib: exploded-image-optimize
 
   build-test-failure-handler: interim-langtools
@@ -1087,7 +1132,7 @@ CLEAN_MODULE_PHASE_TARGETS := $(addprefix clean-, $(foreach m, $(ALL_MODULES), \
 
 # Remove everything, except the output from configure.
 clean: $(CLEAN_DIR_TARGETS)
-	($(CD) $(OUTPUTDIR) && $(RM) -r build*.log*)
+	($(CD) $(OUTPUTDIR) && $(RM) -r build*.log* compile_commands.json)
 	$(ECHO) Cleaned all build artifacts.
 
 clean-docs:

make/ModuleWrapper.gmk

@@ -100,5 +100,9 @@ $(eval $(call SetupCopyFiles, COPY_CONF, \
         $(TARGETS)), \
 ))
 
-all: $(TARGETS) $(COPY_LIBS_TO_BIN) $(COPY_LIBS_TO_LIB) \
-    $(COPY_INCLUDE) $(COPY_CMDS) $(COPY_CONF) $(LINK_LIBS_TO_LIB)
+ifeq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
+  all: $(filter $(MAKESUPPORT_OUTPUTDIR)/compile-commands/%, $(TARGETS))
+else
+  all: $(TARGETS) $(COPY_LIBS_TO_BIN) $(COPY_LIBS_TO_LIB) \
+      $(COPY_INCLUDE) $(COPY_CMDS) $(COPY_CONF) $(LINK_LIBS_TO_LIB)
+endif

make/common/JdkNativeCompilation.gmk

@@ -47,6 +47,29 @@ FindSrcDirsForComponent += \
       $(TOPDIR)/src/$(strip $1)/$(OPENJDK_TARGET_OS_TYPE)/native/$(strip $2) \
       $(TOPDIR)/src/$(strip $1)/share/native/$(strip $2)))
 
+# Find a library
+# Param 1 - module name
+# Param 2 - library name
+# Param 3 - optional subdir for library
+FindLib = \
+    $(call FindLibDirForModule, \
+        $(strip $1))$(strip $3)/$(LIBRARY_PREFIX)$(strip $2)$(SHARED_LIBRARY_SUFFIX)
+
+# Find a static library
+# Param 1 - module name
+# Param 2 - library name
+# Param 3 - optional subdir for library
+FindStaticLib = \
+    $(addprefix $(SUPPORT_OUTPUTDIR)/native/, \
+        $(strip $1)$(strip $3)/$(LIBRARY_PREFIX)$(strip $2)$(STATIC_LIBRARY_SUFFIX))
+
+# If only generating compile_commands.json, make these return empty to avoid
+# declaring dependencies.
+ifeq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
+  FindLib =
+  FindStaticLib =
+endif
+
 GetJavaHeaderDir = \
   $(wildcard $(SUPPORT_OUTPUTDIR)/headers/$(strip $1))
 

make/common/NativeCompilation.gmk

@@ -59,6 +59,29 @@ define GetSymbols
   fi
 endef
 
+################################################################################
+# Creates a recipe that creates a compile_commands.json fragment. Remove any
+# occurences of FIXPATH programs from the command to show the actual invocation.
+#
+# Param 1: Name of file to create
+# Param 2: Working directory
+# Param 3: Source file
+# Param 4: Compile command
+# Param 5: Object name
+################################################################################
+define WriteCompileCommandsFragment
+  $(call LogInfo, Creating compile commands fragment for $(notdir $3))
+  $(call MakeDir, $(dir $1))
+  $(call WriteFile,{ \
+      "directory": "$(strip $2)"$(COMMA) \
+      "file": "$(strip $3)"$(COMMA) \
+      "command": "$(strip $(subst $(DQUOTE),\$(DQUOTE),$(subst \,\\,\
+        $(subst $(FIXPATH),,$4))))"$(COMMA) \
+      "output": "$(strip $5)" \
+    }$(COMMA), \
+    $1)
+endef
+
 ################################################################################
 # Define a native toolchain configuration that can be used by
 # SetupNativeCompilation calls
@@ -203,6 +226,11 @@ define SetupCompileNativeFileBody
   $1_OBJ := $$($$($1_BASE)_OBJECT_DIR)/$$(call replace_with_obj_extension, \
       $$($1_FILENAME))
 
+  # Generate the corresponding compile_commands.json fragment.
+  $1_OBJ_JSON = $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
+      $$(OUTPUTDIR)/,,$$($1_OBJ))).json
+  $$($1_BASE)_ALL_OBJS_JSON += $$($1_OBJ_JSON)
+
   # Only continue if this object file hasn't been processed already. This lets
   # the first found source file override any other with the same name.
   ifeq ($$(findstring $$($1_OBJ), $$($$($1_BASE)_OBJS_SO_FAR)), )
@@ -297,8 +325,15 @@ define SetupCompileNativeFileBody
       $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, $$($1_OBJ).vardeps)
     endif
 
-    $$($1_OBJ): $$($1_SRC_FILE) $$($$($1_BASE)_COMPILE_VARDEPS_FILE) \
-        $$($$($1_BASE)_EXTRA_DEPS) $$($1_VARDEPS_FILE) | $$($$($1_BASE)_BUILD_INFO)
+    $1_OBJ_DEPS := $$($1_SRC_FILE) $$($$($1_BASE)_COMPILE_VARDEPS_FILE) \
+        $$($$($1_BASE)_EXTRA_DEPS) $$($1_VARDEPS_FILE)
+    $1_COMPILE_OPTIONS := $$($1_FLAGS) $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE)
+
+    $$($1_OBJ_JSON): $$($1_OBJ_DEPS)
+	$$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$($1_SRC_FILE), \
+	    $$($1_COMPILER) $$($1_COMPILE_OPTIONS), $$($1_OBJ))
+
+    $$($1_OBJ): $$($1_OBJ_DEPS) | $$($$($1_BASE)_BUILD_INFO)
 	$$(call LogInfo, Compiling $$($1_FILENAME) (for $$($$($1_BASE)_BASENAME)))
 	$$(call MakeDir, $$(@D))
         ifneq ($(TOOLCHAIN_TYPE), microsoft)
@@ -307,13 +342,11 @@ define SetupCompileNativeFileBody
             # object file in the generated deps files. Fixing it with sed. If
             # compiling assembly, don't try this.
 	    $$(call ExecuteWithLog, $$@, \
-	        $$($1_COMPILER) $$($1_FLAGS) $$($1_DEP_FLAG) $$($1_DEP).tmp \
-	            $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE))
+	        $$($1_COMPILER) $$($1_DEP_FLAG) $$($1_DEP).tmp $$($1_COMPILE_OPTIONS))
 	    $(SED) 's|^$$(@F):|$$@:|' $$($1_DEP).tmp > $$($1_DEP)
           else
 	    $$(call ExecuteWithLog, $$@, \
-	        $$($1_COMPILER) $$($1_FLAGS) $$($1_DEP_FLAG) $$($1_DEP) \
-	            $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE))
+	        $$($1_COMPILER) $$($1_DEP_FLAG) $$($1_DEP) $$($1_COMPILE_OPTIONS))
           endif
           # Create a dependency target file from the dependency file.
           # Solution suggested by http://make.mad-scientist.net/papers/advanced-auto-dependency-generation/
@@ -329,8 +362,7 @@ define SetupCompileNativeFileBody
           # on Windows. No need to save exit code from compilation since
           # pipefail is always active on Windows.
 	  $$(call ExecuteWithLog, $$@, \
-	      $$($1_COMPILER) $$($1_FLAGS) -showIncludes \
-	          $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE)) \
+	      $$($1_COMPILER) -showIncludes $$($1_COMPILE_OPTIONS)) \
 	      | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
 	          -e "^$$($1_FILENAME)$$$$" || test "$$$$?" = "1" ; \
 	  $(ECHO) $$@: \\ > $$($1_DEP) ; \
@@ -689,17 +721,25 @@ define SetupNativeCompilationBody
         -include $$($1_PCH_DEP)
         -include $$($1_PCH_DEP_TARGETS)
 
+        $1_PCH_COMMAND := $$($1_CC) $$($1_CFLAGS) $$($1_EXTRA_CFLAGS) $$($1_SYSROOT_CFLAGS) \
+            $$($1_OPT_CFLAGS) -x c++-header -c $(C_FLAG_DEPS) $$($1_PCH_DEP)
+
         $$($1_PCH_FILE): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
 		$$(call LogInfo, Generating precompiled header)
 		$$(call MakeDir, $$(@D))
-		$$(call ExecuteWithLog, $$@, \
-		    $$($1_CC) $$($1_CFLAGS) $$($1_EXTRA_CFLAGS) $$($1_SYSROOT_CFLAGS) \
-		    $$($1_OPT_CFLAGS) \
-		    -x c++-header -c $(C_FLAG_DEPS) $$($1_PCH_DEP) $$< -o $$@)
+		$$(call ExecuteWithLog, $$@, $$($1_PCH_COMMAND) $$< -o $$@)
 		$(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_PCH_DEP) > $$($1_PCH_DEP_TARGETS)
 
         $$($1_ALL_OBJS): $$($1_PCH_FILE)
 
+        # Generate the corresponding compile_commands.json fragment.
+        $1_PCH_FILE_JSON := $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
+            $$(OUTPUTDIR)/,,$$($1_PCH_FILE))).json
+        $1_ALL_OBJS_JSON += $$($1_PCH_FILE_JSON)
+
+        $$($1_PCH_FILE_JSON): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
+		$$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$<, \
+		    $$($1_PCH_COMMAND) $$< -o $$($1_PCH_FILE), $$($1_PCH_FILE))
       endif
     endif
   endif
@@ -1019,6 +1059,10 @@ define SetupNativeCompilationBody
 		  --entitlements $(TOPDIR)/make/data/macosxsigning/entitlements.plist $$@
                 endif
   endif
+
+  ifeq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
+    $1 := $$($1_ALL_OBJS_JSON)
+  endif
 endef
 
 endif # _NATIVE_COMPILATION_GMK

make/conf/jib-profiles.js

@@ -522,7 +522,7 @@ var getJibProfilesProfiles = function (input, common, data) {
         .forEach(function (name) {
             var maketestName = name + "-testmake";
             profiles[maketestName] = concatObjects(profiles[name], testmakeBase);
-            profiles[maketestName].default_make_targets = [ "test-make" ];
+            profiles[maketestName].default_make_targets = [ "test-make", "test-compile-commands" ];
         });
 
     // Profiles for building the zero jvm variant. These are used for verification.