Generate `compilation database "compile_commands.json" // Resolve #2990

HISTORY.rst

@@ -18,6 +18,7 @@ PlatformIO Core 4.0
 
   - Added support for `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ static code analyzer
 
+* Generate `compilation database "compile_commands.json" <https://docs.platformio.org/page/faq.html#compilation-database-compile-commands-json>`_ (`issue #2990 <https://github.com/platformio/platformio-core/issues/2990>`_)
 * Control debug flags and optimization level with a new `debug_build_flags <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-build-flags>`__ option
 * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install <https://docs.platformio.org/page/userguide/platforms/cmd_install.html>`__ command  (`issue #3345 <https://github.com/platformio/platformio-core/issues/3345>`_)
 * Added support for "pythonPackages" in `platform.json <https://docs.platformio.org/page/platforms/creating_platform.html#manifest-file-platform-json>`__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed)

platformio/builder/main.py

@@ -72,6 +72,7 @@
     BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
     BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
     BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
+    COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
@@ -134,6 +135,10 @@
 elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     click.echo("Verbose mode can be enabled via `-v, --verbose` option")
 
+# Dynamically load dependent tools
+if "compiledb" in COMMAND_LINE_TARGETS:
+    env.Tool("compilation_db")
+
 if not isdir(env.subst("$BUILD_DIR")):
     makedirs(env.subst("$BUILD_DIR"))
 
@@ -171,6 +176,9 @@
     Default(_new_targets)
     Default("checkprogsize")
 
+if "compiledb" in COMMAND_LINE_TARGETS:
+    env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH"))
+
 # Print configured protocols
 env.AddPreAction(
     ["upload", "program"],

platformio/builder/tools/compilation_db.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+# Copyright 2015 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-argument, protected-access, unused-variable, import-error
+# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py
+
+from __future__ import absolute_import
+
+import itertools
+import json
+
+import SCons
+
+from platformio.builder.tools.platformio import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT
+
+# Implements the ability for SCons to emit a compilation database for the MongoDB project. See
+# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation
+# database is, and why you might want one. The only user visible entry point here is
+# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that
+# should hold the compilation database, otherwise, the file defaults to compile_commands.json,
+# which is the name that most clang tools search for by default.
+
+# TODO: Is there a better way to do this than this global? Right now this exists so that the
+# emitter we add can record all of the things it emits, so that the scanner for the top level
+# compilation database can access the complete list, and also so that the writer has easy
+# access to write all of the files. But it seems clunky. How can the emitter and the scanner
+# communicate more gracefully?
+__COMPILATION_DB_ENTRIES = []
+
+
+# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even
+# integrate with the cache, but there doesn't seem to be much call for it.
+class __CompilationDbNode(SCons.Node.Python.Value):
+    def __init__(self, value):
+        SCons.Node.Python.Value.__init__(self, value)
+        self.Decider(changed_since_last_build_node)
+
+
+def changed_since_last_build_node(child, target, prev_ni, node):
+    """ Dummy decider to force always building"""
+    return True
+
+
+def makeEmitCompilationDbEntry(comstr):
+    """
+    Effectively this creates a lambda function to capture:
+    * command line
+    * source
+    * target
+    :param comstr: unevaluated command line
+    :return: an emitter which has captured the above
+    """
+    user_action = SCons.Action.Action(comstr)
+
+    def EmitCompilationDbEntry(target, source, env):
+        """
+        This emitter will be added to each c/c++ object build to capture the info needed
+        for clang tools
+        :param target: target node(s)
+        :param source: source node(s)
+        :param env: Environment for use building this node
+        :return: target(s), source(s)
+        """
+
+        dbtarget = __CompilationDbNode(source)
+
+        entry = env.__COMPILATIONDB_Entry(
+            target=dbtarget,
+            source=[],
+            __COMPILATIONDB_UTARGET=target,
+            __COMPILATIONDB_USOURCE=source,
+            __COMPILATIONDB_UACTION=user_action,
+            __COMPILATIONDB_ENV=env,
+        )
+
+        # TODO: Technically, these next two lines should not be required: it should be fine to
+        # cache the entries. However, they don't seem to update properly. Since they are quick
+        # to re-generate disable caching and sidestep this problem.
+        env.AlwaysBuild(entry)
+        env.NoCache(entry)
+
+        __COMPILATION_DB_ENTRIES.append(dbtarget)
+
+        return target, source
+
+    return EmitCompilationDbEntry
+
+
+def CompilationDbEntryAction(target, source, env, **kw):
+    """
+    Create a dictionary with evaluated command line, target, source
+    and store that info as an attribute on the target
+    (Which has been stored in __COMPILATION_DB_ENTRIES array
+    :param target: target node(s)
+    :param source: source node(s)
+    :param env: Environment for use building this node
+    :param kw:
+    :return: None
+    """
+
+    command = env["__COMPILATIONDB_UACTION"].strfunction(
+        target=env["__COMPILATIONDB_UTARGET"],
+        source=env["__COMPILATIONDB_USOURCE"],
+        env=env["__COMPILATIONDB_ENV"],
+    )
+
+    entry = {
+        "directory": env.Dir("#").abspath,
+        "command": command,
+        "file": str(env["__COMPILATIONDB_USOURCE"][0]),
+    }
+
+    target[0].write(entry)
+
+
+def WriteCompilationDb(target, source, env):
+    entries = []
+
+    for s in __COMPILATION_DB_ENTRIES:
+        entries.append(s.read())
+
+    with open(str(target[0]), "w") as target_file:
+        json.dump(
+            entries, target_file, sort_keys=True, indent=4, separators=(",", ": ")
+        )
+
+
+def ScanCompilationDb(node, env, path):
+    return __COMPILATION_DB_ENTRIES
+
+
+def generate(env, **kwargs):
+
+    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
+
+    env["COMPILATIONDB_COMSTR"] = kwargs.get(
+        "COMPILATIONDB_COMSTR", "Building compilation database $TARGET"
+    )
+
+    components_by_suffix = itertools.chain(
+        itertools.product(
+            [".%s" % ext for ext in SRC_C_EXT],
+            [
+                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"),
+                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"),
+            ],
+        ),
+        itertools.product(
+            [".%s" % ext for ext in SRC_CXX_EXT],
+            [
+                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"),
+                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"),
+            ],
+        ),
+        itertools.product(
+            [".%s" % ext for ext in SRC_ASM_EXT],
+            [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")],
+        ),
+    )
+
+    for entry in components_by_suffix:
+        suffix = entry[0]
+        builder, base_emitter, command = entry[1]
+
+        # Assumes a dictionary emitter
+        emitter = builder.emitter[suffix]
+        builder.emitter[suffix] = SCons.Builder.ListEmitter(
+            [emitter, makeEmitCompilationDbEntry(command)]
+        )
+
+    env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder(
+        action=SCons.Action.Action(CompilationDbEntryAction, None),
+    )
+
+    env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder(
+        action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"),
+        target_scanner=SCons.Scanner.Scanner(
+            function=ScanCompilationDb, node_class=None
+        ),
+    )
+
+    def CompilationDatabase(env, target):
+        result = env.__COMPILATIONDB_Database(target=target, source=[])
+
+        env.AlwaysBuild(result)
+        env.NoCache(result)
+
+        return result
+
+    env.AddMethod(CompilationDatabase, "CompilationDatabase")
+
+
+def exists(env):
+    return True

platformio/builder/tools/piolib.py

@@ -366,7 +366,7 @@ def _get_found_includes(  # pylint: disable=too-many-branches
                 if not fs.path_endswith_ext(_h_path, piotool.SRC_HEADER_EXT):
                     continue
                 _f_part = _h_path[: _h_path.rindex(".")]
-                for ext in piotool.SRC_C_EXT:
+                for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT:
                     if not isfile("%s.%s" % (_f_part, ext)):
                         continue
                     _c_path = self.env.File("%s.%s" % (_f_part, ext))

platformio/builder/tools/platformio.py

@@ -31,8 +31,10 @@
 from platformio.util import pioversion_to_intstr
 
 SRC_HEADER_EXT = ["h", "hpp"]
-SRC_C_EXT = ["c", "cc", "cpp"]
-SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
+SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
+SRC_C_EXT = ["c"]
+SRC_CXX_EXT = ["cc", "cpp", "cxx", "c++"]
+SRC_BUILD_EXT = SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT
 SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep]