[report-converter] Kernel-Doc Parser

docs/README.md

@@ -109,6 +109,7 @@ The following tools are supported:
 |                | [Facebook Infer](/tools/report-converter/README.md#facebook-infer)    |
 |                | [Coccinelle](/tools/report-converter/README.md#coccinelle)    |
 |                | [Smatch](/tools/report-converter/README.md#smatch)    |
+|                | [Kernel-Doc](/tools/report-converter/README.md#kernel-doc)    |
 | **Java**       | [SpotBugs](/tools/report-converter/README.md#spotbugs)    |
 |                | [Facebook Infer](/tools/report-converter/README.md#fbinfer)    |
 | **Python**     | [Pylint](/tools/report-converter/README.md#pylint)    |

docs/supported_code_analyzers.md

@@ -17,6 +17,7 @@ CodeChecker result directory which can be stored to a CodeChecker server.
 |                | [Facebook Infer](/tools/report-converter/README.md#fbinfer)    | ✓ |
 |                | [Coccinelle](/tools/report-converter/README.md#coccinelle)   | ✓ |
 |                | [Smatch](/tools/report-converter/README.md#smatch)   | ✓ |
+|                | [Kernel-Doc](/tools/report-converter/README.md#kernel-doc)   | ✓ |
 | **Java**       | [FindBugs](http://findbugs.sourceforge.net/)    | ✗ |
 |                | [SpotBugs](/tools/report-converter/README.md#spotbugs)    | ✓ |
 |                | [Facebook Infer](/tools/report-converter/README.md#fbinfer)    | ✓ |

tools/report-converter/README.md

@@ -22,6 +22,7 @@ a CodeChecker server.
 * [Markdownlint](#markdownlint)
 * [Coccinelle](#coccinelle)
 * [Smatch](#smatch)
+* [Kernel-Doc](#kernel-doc)
 * [License](#license)
 
 ## Install guide
@@ -54,8 +55,9 @@ optional arguments:
                         report directory files.
   -t TYPE, --type TYPE  Specify the format of the code analyzer output.
                         Currently supported output types are: asan, clang-
-                        tidy, cppcheck, eslint, fbinfer, golint, msan,
-                        pyflakes, pylint, spotbugs, tsan, tslint, ubsan.
+                        tidy, coccinelle, cppcheck, eslint, fbinfer, golint,
+                        kernel-doc, msan, pyflakes, pylint, smatch, spotbugs, 
+                        tsan, tslint, ubsan.
   --meta [META [META ...]]
                         Metadata information which will be stored alongside
                         the run when the created report directory will be
@@ -86,6 +88,7 @@ Supported analyzers:
   eslint - ESLint, https://eslint.org/
   fbinfer - Facebook Infer, https://fbinfer.com
   golint - Golint, https://github.com/golang/lint
+  kernel-doc - Kernel-Doc, https://github.com/torvalds/linux/blob/master/scripts/kernel-doc
   mdl - Markdownlint, https://github.com/markdownlint/markdownlint
   msan - MemorySanitizer, https://clang.llvm.org/docs/MemorySanitizer.html
   pyflakes - Pyflakes, https://github.com/PyCQA/pyflakes
@@ -441,6 +444,33 @@ report-converter -t smatch -o ./codechecker_smatch_reports ./smatch_warns.txt
 CodeChecker store ./codechecker_smatch_reports -n smatch
 ```
 
+## [Kernel-Doc](https://github.com/torvalds/linux/blob/master/scripts/kernel-doc)
+[Kernel-Doc](https://github.com/torvalds/linux/blob/master/scripts/kernel-doc) structure is extracted 
+from the comments, and proper Sphinx C Domain function and type descriptions with anchors are generated 
+from them. The descriptions are filtered for special kernel-doc highlights and cross-references.
+
+The recommended way of running Kernel-Doc is to redirect the output to a file and
+give this file to the report converter tool.
+
+The following example shows you how to run Kernel-Doc on kernel sources 
+and store the results found by Kernel-Doc to the CodeChecker database.
+```sh
+# Change Directory to your project
+cd path/to/linux/kernel/repository
+
+# Run Kernel-Doc
+# Note: The output of the following command will be both of sphinx and kernel-doc, 
+# but the parser will parse only kernel-doc output
+make htmldocs 2>&1 | tee kernel-docs.out
+
+# Use 'report-converter' to create a CodeChecker report directory from the
+# analyzer result of Kernel-Doc
+report-converter -t kernel-doc -o ./codechecker_kernel_doc_reports ./sphinx_output.out
+
+# Store the Kernel-Doc reports with CodeChecker.
+CodeChecker store ./codechecker_kernel_doc_reports -n kernel-doc
+```
+
 ## License
 
 The project is licensed under Apache License v2.0 with LLVM Exceptions.

tools/report-converter/codechecker_report_converter/cli.py

@@ -56,6 +56,8 @@
     CoccinelleAnalyzerResult  # noqa
 from codechecker_report_converter.smatch.analyzer_result import \
     SmatchAnalyzerResult  # noqa
+from codechecker_report_converter.kerneldoc.analyzer_result import \
+    KernelDocAnalyzerResult  # noqa
 
 
 LOG = logging.getLogger('ReportConverter')
@@ -92,7 +94,8 @@ class RawDescriptionDefaultHelpFormatter(
     SpotBugsAnalyzerResult.TOOL_NAME: SpotBugsAnalyzerResult,
     MarkdownlintAnalyzerResult.TOOL_NAME: MarkdownlintAnalyzerResult,
     CoccinelleAnalyzerResult.TOOL_NAME: CoccinelleAnalyzerResult,
-    SmatchAnalyzerResult.TOOL_NAME: SmatchAnalyzerResult
+    SmatchAnalyzerResult.TOOL_NAME: SmatchAnalyzerResult,
+    KernelDocAnalyzerResult.TOOL_NAME: KernelDocAnalyzerResult
 }
 
 supported_metadata_keys = ["analyzer_command", "analyzer_version"]

tools/report-converter/codechecker_report_converter/kerneldoc/__init__.py

@@ -0,0 +1,7 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------

tools/report-converter/codechecker_report_converter/kerneldoc/analyzer_result.py

@@ -0,0 +1,36 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+from codechecker_report_converter.analyzer_result import AnalyzerResult
+
+from .output_parser import KernelDocParser
+from ..plist_converter import PlistConverter
+
+
+class KernelDocAnalyzerResult(AnalyzerResult):
+    """ Transform analyzer result of kernel-docs. """
+
+    TOOL_NAME = 'kernel-doc'
+    NAME = 'Kernel-Doc'
+    URL = 'https://github.com/torvalds/linux/blob/master/scripts/kernel-doc'
+
+    def parse(self, analyzer_result):
+        """ Creates plist files from the given analyzer result to the given
+        output directory.
+        """
+        parser = KernelDocParser(analyzer_result)
+
+        content = self._get_analyzer_result_file_content(analyzer_result)
+        if not content:
+            return
+
+        messages = parser.parse_messages(content)
+
+        plist_converter = PlistConverter(self.TOOL_NAME)
+        plist_converter.add_messages(messages)
+        return plist_converter.get_plist_results()

tools/report-converter/codechecker_report_converter/kerneldoc/output_parser.py

@@ -0,0 +1,61 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+import logging
+import os
+import re
+
+from ..output_parser import BaseParser, Message
+LOG = logging.getLogger('ReportConverter')
+
+
+class KernelDocParser(BaseParser):
+    """
+    Parser for Kernel-Docs Output
+    """
+
+    def __init__(self, analyzer_result):
+        super(KernelDocParser, self).__init__()
+
+        self.analyzer_result = analyzer_result
+
+        self.message_line_re = re.compile(
+            # File path starting with '.' and followed by a ':'.
+            r'^(?P<path>\.[\S ]+?):'
+            # Line number followed by a ':'.
+            r'(?P<line>\d+?):\s'
+            # Message.
+            r'(?P<message>[\S \t]+)\s*')
+
+    def parse_message(self, it, line):
+        """
+        Actual Parsing function for the given line
+        It is expected that each line contains a seperate report
+        """
+        match = self.message_line_re.match(line)
+
+        if match is None:
+            return None, next(it)
+
+        file_path = os.path.normpath(
+            os.path.join(os.path.dirname(self.analyzer_result),
+                         match.group('path')))
+
+        checker_name = None
+
+        message = Message(
+            file_path,
+            int(match.group('line')),
+            0,
+            match.group('message').strip(),
+            checker_name)
+
+        try:
+            return message, next(it)
+        except StopIteration:
+            return message, ''

tools/report-converter/tests/unit/kerneldoc_output_test_files/files/sample.c

@@ -0,0 +1,9 @@
+// Expected warning: Kenel-Doc.
+#include <stdio.h>
+
+int test_sample(int a)
+{
+    int x;
+    x = some_function(a);
+    return x;
+}

tools/report-converter/tests/unit/kerneldoc_output_test_files/sample.expected.plist

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>diagnostics</key>
+	<array>
+		<dict>
+			<key>category</key>
+			<string>unknown</string>
+			<key>check_name</key>
+			<string>kernel-doc</string>
+			<key>description</key>
+			<string>warning: no structured comments found</string>
+			<key>issue_hash_content_of_line_in_context</key>
+			<string>9adbf7ff45e7ba1c01b338d4fc2b4c4b</string>
+			<key>location</key>
+			<dict>
+				<key>col</key>
+				<integer>0</integer>
+				<key>file</key>
+				<integer>0</integer>
+				<key>line</key>
+				<integer>1</integer>
+			</dict>
+			<key>path</key>
+			<array>
+				<dict>
+					<key>depth</key>
+					<integer>0</integer>
+					<key>kind</key>
+					<string>event</string>
+					<key>location</key>
+					<dict>
+						<key>col</key>
+						<integer>0</integer>
+						<key>file</key>
+						<integer>0</integer>
+						<key>line</key>
+						<integer>1</integer>
+					</dict>
+					<key>message</key>
+					<string>warning: no structured comments found</string>
+				</dict>
+			</array>
+			<key>type</key>
+			<string>kernel-doc</string>
+		</dict>
+	</array>
+	<key>files</key>
+	<array>
+		<string>files/sample.c</string>
+	</array>
+	<key>metadata</key>
+	<dict>
+		<key>analyzer</key>
+		<dict>
+			<key>name</key>
+			<string>kernel-doc</string>
+		</dict>
+		<key>generated_by</key>
+		<dict>
+			<key>name</key>
+			<string>report-converter</string>
+			<key>version</key>
+			<string>x.y.z</string>
+		</dict>
+	</dict>
+</dict>
+</plist>

tools/report-converter/tests/unit/kerneldoc_output_test_files/sample.out

@@ -0,0 +1 @@
+./files/sample.c:1: warning: no structured comments found

tools/report-converter/tests/unit/test_kerneldoc_parser.py

@@ -0,0 +1,79 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+"""
+This module tests the correctness of the KernelDocumentationAnalyzerResult,
+which used in sequence transform Kernel-Doc output to a plist file.
+"""
+
+
+import os
+import plistlib
+import shutil
+import tempfile
+import unittest
+
+
+from codechecker_report_converter.kerneldoc.analyzer_result import \
+    KernelDocAnalyzerResult
+
+
+class KernelDocAnalyzerResultTestCase(unittest.TestCase):
+    """ Test the output of the KernelDocAnalyzerResult. """
+
+    def setUp(self):
+        """ Setup the test. """
+        self.analyzer_result = KernelDocAnalyzerResult()
+        self.cc_result_dir = tempfile.mkdtemp()
+        self.test_files = os.path.join(os.path.dirname(__file__),
+                                       'kerneldoc_output_test_files')
+
+    def tearDown(self):
+        """ Clean temporary directory. """
+        shutil.rmtree(self.cc_result_dir)
+
+    def test_no_kerneldoc_output_file(self):
+        """ Test transforming single C file. """
+        analyzer_result = os.path.join(self.test_files, 'files',
+                                       'sample.c')
+
+        ret = self.analyzer_result.transform(analyzer_result,
+                                             self.cc_result_dir)
+        self.assertFalse(ret)
+
+    def test_transform_dir(self):
+        """ Test transforming a directory. """
+        analyzer_result = os.path.join(self.test_files)
+
+        ret = self.analyzer_result.transform(analyzer_result,
+                                             self.cc_result_dir)
+        self.assertFalse(ret)
+
+    def test_transform_single_file(self):
+        """ Test transforming single output file. """
+        analyzer_result = os.path.join(self.test_files, 'sample.out')
+        self.analyzer_result.transform(analyzer_result, self.cc_result_dir)
+
+        plist_file = os.path.join(self.cc_result_dir,
+                                  'sample.c_kernel-doc.plist')
+
+        with open(plist_file, mode='rb') as pfile:
+            res = plistlib.load(pfile)
+
+            # Use relative path for this test.
+            res['files'][0] = os.path.join('files', 'sample.c')
+
+            self.assertTrue(res['metadata']['generated_by']['version'])
+            res['metadata']['generated_by']['version'] = "x.y.z"
+
+        plist_file = os.path.join(self.test_files,
+                                  'sample.expected.plist')
+        with open(plist_file, mode='rb') as pfile:
+            exp = plistlib.load(pfile)
+
+        self.assertEqual(res, exp)