Merge pull request #3815 from bruntib/multiroot

Multiroot

analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py

@@ -451,7 +451,6 @@ def construct_config_handler(cls, args):
             handler.ctu_on_demand = \
                 'ctu_ast_mode' in args and \
                 args.ctu_ast_mode == 'parse-on-demand'
-            handler.log_file = args.logfile
 
         try:
             with open(args.clangsa_args_cfg_file, 'r', encoding='utf8',

analyzer/codechecker_analyzer/analyzers/clangsa/config_handler.py

@@ -28,7 +28,6 @@ def __init__(self, environ):
         super(ClangSAConfigHandler, self).__init__()
         self.ctu_dir = ''
         self.ctu_on_demand = False
-        self.log_file = ''
         self.enable_z3 = False
         self.enable_z3_refutation = False
         self.environ = environ

analyzer/codechecker_analyzer/arg.py

@@ -9,6 +9,7 @@
 
 
 import argparse
+import os
 
 
 class OrderedCheckersAction(argparse.Action):
@@ -71,3 +72,18 @@ def __call__(self, parser, namespace, value, option_string=None):
             if flag in dest:
                 dest.remove(flag)
             dest.append(flag)
+
+
+def existing_abspath(path: str) -> str:
+    """
+    This function can be used at "type" argument of argparse.add_argument()
+    function. It returns the absolute path of the given path if exists
+    otherwise raises an argparse.ArgumentTypeError which constitutes in a
+    graceful error message automatically by argparse module.
+    """
+    path = os.path.abspath(path)
+
+    if not os.path.exists(path):
+        raise argparse.ArgumentTypeError(f"File doesn't exist: {path}")
+
+    return path

analyzer/codechecker_analyzer/cmd/analyze.py

@@ -16,18 +16,18 @@
 import multiprocessing
 import os
 import re
-import shlex
 import shutil
 import sys
 
 from typing import List
 
 from tu_collector import tu_collector
 
-from codechecker_analyzer import analyzer, analyzer_context
+from codechecker_analyzer import analyzer, analyzer_context, \
+    compilation_database
 from codechecker_analyzer.analyzers import analyzer_types, clangsa
 from codechecker_analyzer.arg import \
-        OrderedCheckersAction, OrderedConfigAction
+        OrderedCheckersAction, OrderedConfigAction, existing_abspath
 from codechecker_analyzer.buildlog import log_parser
 
 from codechecker_common import arg, logger, cmd_config
@@ -153,12 +153,12 @@ def add_arguments_to_parser(parser):
     Add the subcommand's arguments to the given argparse.ArgumentParser.
     """
 
-    parser.add_argument('logfile',
-                        type=str,
-                        help="Path to the JSON compilation command database "
-                             "files which were created during the build. "
-                             "The analyzers will check only the files "
-                             "registered in these build databases.")
+    parser.add_argument('input',
+                        type=existing_abspath,
+                        help="The input of the analysis can be either a "
+                             "compilation database JSON file, a path to a "
+                             "source file or a path to a directory containing "
+                             "source files.")
 
     parser.add_argument('-j', '--jobs',
                         type=int,
@@ -854,26 +854,11 @@ def __get_result_source_files(metadata):
     return result_src_files
 
 
-def __change_args_to_command_in_comp_db(compile_commands):
-    """
-    In CodeChecker we support compilation databases where the JSON object of a
-    build action contains "file", "directory" and "command" fields. However,
-    compilation databases from intercept build contain "arguments" instead of
-    "command" which is a list of command-line arguments instead of the same
-    command as a single string. This function make this appropriate conversion.
-    """
-    for cc in compile_commands:
-        if 'command' not in cc:
-            # TODO: shlex.join(cmd) would be more elegant after upgrading to
-            # Python 3.8.
-            cc['command'] = ' '.join(map(shlex.quote, cc['arguments']))
-            del cc['arguments']
-
-
 def main(args):
     """
-    Perform analysis on the given logfiles and store the results in a machine-
-    readable format.
+    Perform analysis on the given inputs. Possible inputs are a compilation
+    database, a source file or the path of a project. The analysis results are
+    stored to a report directory given by -o flag.
     """
     logger.setup_logger(args.verbose if 'verbose' in args else None)
 
@@ -893,10 +878,6 @@ def main(args):
         LOG.error(fnerr)
         sys.exit(1)
 
-    if not os.path.exists(args.logfile):
-        LOG.error("The specified logfile '%s' does not exist!", args.logfile)
-        sys.exit(1)
-
     args.output_path = os.path.abspath(args.output_path)
     if os.path.exists(args.output_path) and \
             not os.path.isdir(args.output_path):
@@ -937,10 +918,10 @@ def main(args):
             sys.exit(1)
         compiler_info_file = args.compiler_info_file
 
-    compile_commands = load_json(args.logfile)
+    compile_commands = \
+        compilation_database.gather_compilation_database(args.input)
     if compile_commands is None:
         sys.exit(1)
-    __change_args_to_command_in_comp_db(compile_commands)
 
     # Process the skip list if present.
     skip_handlers = __get_skip_handlers(args, compile_commands)
@@ -1097,16 +1078,8 @@ def main(args):
 
     # WARN: store command will search for this file!!!!
     compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json')
-    try:
-        source = os.path.abspath(args.logfile)
-        target = os.path.abspath(compile_cmd_json)
-
-        if source != target:
-            shutil.copyfile(source, target)
-    except shutil.Error:
-        LOG.debug("Compilation database JSON file is the same.")
-    except Exception:
-        LOG.debug("Copying compilation database JSON file failed.")
+    with open(compile_cmd_json, 'w', encoding="utf-8", errors="ignore") as f:
+        json.dump(compile_commands, f, indent=2)
 
     try:
         # pylint: disable=no-name-in-module

analyzer/codechecker_analyzer/cmd/check.py

@@ -800,7 +800,7 @@ def __update_if_key_exists(source, target, key):
 
         # --- Step 2.: Perform the analysis.
         analyze_args = argparse.Namespace(
-            logfile=logfile,
+            input=logfile,
             output_path=output_dir,
             output_format='plist',
             jobs=args.jobs,

analyzer/codechecker_analyzer/compilation_database.py

@@ -0,0 +1,209 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+"""
+Utilities for compilation database handling.
+"""
+
+
+import os
+import shlex
+from typing import Callable, Dict, List, Optional
+
+from codechecker_common.util import load_json
+
+
+# For details see
+# https://gcc.gnu.org/onlinedocs/gcc-12.2.0/gcc/Overall-Options.html#Overall-Options
+C_CPP_OBJC_OBJCPP_EXTS = [
+    '.c', '.i', '.ii', '.m', '.mi', '.mm', '.M', '.mii',
+    '.cc', '.cp', '.cxx', '.cpp', '.CPP', 'c++', '.C',
+    '.hh', '.H', '.hp', '.hxx', '.hpp', '.HPP', '.h++', '.tcc'
+]
+
+
+# Compilation database is conventionally stored in this file. Many Clang-based
+# tools rely on this file name, and CMake exports this too.
+COMPILATION_DATABASE = "compile_commands.json"
+
+
+def find_closest_compilation_database(path: str) -> Optional[str]:
+    """
+    Traverse the parent directories of the given path and find the closest
+    compile_commands.json. If no JSON file exists with this name up to the root
+    then None returns.
+    The path of the first compilation database is returned even if it doesn't
+    contain a corresponding entry for the given source file in "path".
+    """
+    path = os.path.abspath(path)
+    root = os.path.abspath(os.sep)
+
+    while True:
+        path = os.path.dirname(path)
+        compile_commands_json = os.path.join(path, COMPILATION_DATABASE)
+
+        if os.path.isfile(compile_commands_json):
+            return compile_commands_json
+
+        if path == root:
+            break
+
+
+def change_args_to_command_in_comp_db(compile_commands: List[Dict]):
+    """
+    In CodeChecker we support compilation databases where the JSON object of a
+    build action contains "file", "directory" and "command" fields. However,
+    compilation databases from intercept build contain "arguments" instead of
+    "command" which is a list of command-line arguments instead of the same
+    command as a single string. This function make this appropriate conversion.
+    """
+    for cc in compile_commands:
+        if 'command' not in cc:
+            # TODO: shlex.join(cmd) would be more elegant after upgrading to
+            # Python 3.8.
+            cc['command'] = ' '.join(map(shlex.quote, cc['arguments']))
+            del cc['arguments']
+
+
+def find_all_compilation_databases(path: str) -> List[str]:
+    """
+    Collect all compilation database paths that may be relevant for the source
+    files at the given path. This means compilation databases anywhere under
+    this path and in the parent directories up to the root.
+    """
+    dirs = []
+
+    for root, _, files in os.walk(path):
+        if COMPILATION_DATABASE in files:
+            dirs.append(os.path.join(root, COMPILATION_DATABASE))
+
+    path = os.path.abspath(path)
+    root = os.path.abspath(os.sep)
+
+    while True:
+        path = os.path.dirname(path)
+        compile_commands_json = os.path.join(path, COMPILATION_DATABASE)
+
+        if os.path.isfile(compile_commands_json):
+            dirs.append(compile_commands_json)
+
+        if path == root:
+            break
+
+    return dirs
+
+
+def build_action_describes_file(file_path: str) -> Callable[[Dict], bool]:
+    """
+    Returns a function which checks whether a build action belongs to the
+    given file_path. This returned function can be used for filtering
+    build actions in a compilation database to find build actions belonging
+    to the given source file.
+    """
+    def describes_file(build_action) -> bool:
+        return os.path.abspath(file_path) == os.path.abspath(
+            os.path.join(build_action['directory'], build_action['file']))
+    return describes_file
+
+
+def is_c_lang_source_file(source_file_path: str) -> bool:
+    """
+    A file is candidate if a build action may belong to it in some
+    compilation database, i.e. it is a C/C++/Obj-C source file.
+    """
+    return os.path.isfile(source_file_path) and \
+        os.path.splitext(source_file_path)[1] in C_CPP_OBJC_OBJCPP_EXTS
+
+
+def find_build_actions_for_file(file_path: str) -> List[Dict]:
+    """
+    Find the corresponding compilation database belonging to the given
+    source file and return a list of build actions that describe its
+    compilation.
+    """
+    comp_db = find_closest_compilation_database(file_path)
+
+    if comp_db is None:
+        return []
+
+    return list(filter(
+        build_action_describes_file(file_path),
+        load_json(comp_db)))
+
+
+def gather_compilation_database(analysis_input: str) -> Optional[List[Dict]]:
+    """
+    Return a compilation database that describes the build of the given
+    analysis_input:
+
+    - If analysis_input is a compilation database JSON file then its entries
+      return.
+    - If analysis_input is a C/C++/Obj-C source file then the corresponding
+      build command is found from the compilation database. Only the innermost
+      compilation database is checked (see find_closest_compilation_database()
+      for details).
+    - If analysis_input is a directory then a compilation database with the
+      build commands of all C/C++/Obj-C files in it return.
+
+    If none of these apply (e.g. analysis_input is a Python source file which
+    doesn't have a compilation database) then None returns.
+    """
+    def __select_compilation_database(
+        comp_db_paths: List[str],
+        source_file: str
+    ) -> Optional[str]:
+        """
+        Helper function for selecting the corresponding compilation database
+        for the given source file, i.e. the compilation database in the closest
+        parent directory, even if it doesn't contain a build action belonging
+        to this file.
+        """
+        comp_db_paths = list(map(os.path.dirname, comp_db_paths))
+        longest = os.path.commonpath(comp_db_paths + [source_file])
+        if longest in comp_db_paths:
+            return os.path.join(longest, COMPILATION_DATABASE)
+
+    # Case 1: analysis_input is a compilation database JSON file.
+
+    build_actions = load_json(analysis_input, display_warning=False)
+
+    if build_actions is not None:
+        pass
+
+    # Case 2: analysis_input is a C/C++/Obj-C source file.
+
+    elif is_c_lang_source_file(analysis_input):
+        build_actions = find_build_actions_for_file(analysis_input)
+
+    # Case 3: analysis_input is a directory.
+
+    elif os.path.isdir(analysis_input):
+        compilation_database_files = \
+            find_all_compilation_databases(analysis_input)
+
+        build_actions = []
+
+        for comp_db_file in compilation_database_files:
+            comp_db = load_json(comp_db_file)
+
+            if not comp_db:
+                continue
+
+            for ba in comp_db:
+                file_path = os.path.join(ba["directory"], ba["file"])
+
+                if os.path.commonpath([file_path, analysis_input]) \
+                        == analysis_input and __select_compilation_database(
+                        compilation_database_files,
+                        file_path) == comp_db_file:
+                    build_actions.append(ba)
+
+    # Compilation database transformation.
+    if build_actions is not None:
+        change_args_to_command_in_comp_db(build_actions)
+
+    return build_actions