Add reporter for outputing Reviewdog JSONL format

This reporter will convert linter output to a standardised format that can
easily be read by reviewdog.

I have decided to use the JSONL format over the JSON format as it will be
simpler to join together mutltiple files before feeding into review dog.

CHANGELOG.md

@@ -8,6 +8,7 @@ Note: Can be used with `nvuillam/mega-linter@insiders` in your GitHub Action meg
 - Use Python virtual-environment in dev-dependencies shell example
 - Fix #367 : Display editorconfig-checker version
 - Fix #379 : New configuration FAIL_IF_MISSING_LINTER_IN_FLAVOR
+- Add Reviewdog JSONL Reporter
 
 - Linter versions upgrades
   - [flake8](https://flake8.pycqa.org) from 3.8.4 to **3.9.0** on 2021-03-15

docs/CHANGELOG.md

@@ -9,6 +9,7 @@ Note: Can be used with `nvuillam/mega-linter@insiders` in your GitHub Action meg
 - Use Python virtual-environment in dev-dependencies shell example
 - Fix #367 : Display editorconfig-checker version
 - Fix #379 : New configuration FAIL_IF_MISSING_LINTER_IN_FLAVOR
+- Add Reviewdog JSONL Reporter
 
 - Linter versions upgrades
   - [flake8](https://flake8.pycqa.org) from 3.8.4 to **3.9.0** on 2021-03-15

megalinter/descriptors/python.megalinter-descriptor.yml

@@ -42,6 +42,7 @@ linters:
       vscode:
         - name: Native Support
           url: https://code.visualstudio.com/docs/python/linting#_pylint
+
   # BLACK
   - linter_name: black
     name: PYTHON_BLACK
@@ -87,6 +88,17 @@ linters:
         - name: VsCode Python Extension
           url: https://marketplace.visualstudio.com/items?itemName=ms-python.python
 
+    reviewdog_processor:
+      class_name: UnifiedDiffs
+      init_params:
+        file_header_regex: ^Linter log for (.+)$
+        ignored_line_regexes:
+          - ^would reformat \S+$
+          - ^Oh no! .*$
+          - ^\d+ file would be reformatted.$
+          - ^All done! .*$
+          - ^\d+ file would be left unchanged.$
+
   # FLAKE8
   - linter_name: flake8
     name: PYTHON_FLAKE8
@@ -115,6 +127,10 @@ linters:
       vscode:
         - name: Native Support
           url: https://code.visualstudio.com/docs/python/linting#_flake8
+    reviewdog_processor:
+      class_name: Regex
+      init_params:
+        regex: ^(?P<path>[^:]+):(?P<line>\d+):(?P<column>\d+):(?P<message>.+)$
 
   # ISORT
   - linter_name: isort

megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json

@@ -914,6 +914,28 @@
             "title": "Linter configuration key",
             "type": "string"
           },
+          "reviewdog_processor": {
+            "$id": "#/properties/linters/items/properties/reviewdog_processor",
+            "description": "Reviewdog processor to convert linter output to RDJSONL format",
+            "properties": {
+              "class_name": {
+                "$id": "#/properties/linters/items/properties/reviewdog_processor/class_name",
+                "description": "Subclass of RdjsonlConvertor to use to process linter output",
+                "examples": [
+                  "Regex",
+                  "UnifiedDiffs"
+                ],
+                "type": "string"
+              },
+              "init_params": {
+                "$id": "#/properties/linters/items/properties/reviewdog_processor/init_params",
+                "description": "Initi parameters for initialising subclass of RdjsonlConvertor",
+                "type": "object"
+              }
+            },
+            "title": "Reviewdog processor",
+            "type": "object"
+          },
           "test_folder": {
             "$id": "#/properties/linters/items/test_folder",
             "description": "Test folder containing _good_ and _bad_ files, if different from parent descriptor test_folder",

megalinter/reporters/ReviewdogJsonlLinterReporter.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""
+Reviewdog JSONL linter reporter
+See https://github.com/reviewdog/reviewdog/tree/master/proto/rdf#rdjsonl for details
+"""
+import json
+import logging
+import os
+import re
+
+from megalinter import Reporter, config, utils
+
+
+class Location(dict):
+    def __init__(self, line, column) -> None:
+        super().__init__(line=line, column=column)
+
+
+class Suggestion(dict):
+    def __init__(self, suggestion_text, start, end) -> None:
+        super().__init__(text=suggestion_text, range={"start": start, "end": end})
+
+
+class Rdjsonl(dict):
+    def __init__(self, file, message, start, end=None, severity="ERROR", suggestions=None) -> None:
+        if suggestions is None:
+            suggestions = []
+        location = {"range": {"start": start, "end": end}, "path": file}
+        super().__init__(message=message, location=location, severity=severity, suggestions=suggestions)
+
+
+class ReviewdogJsonlLinterReporter(Reporter):
+    name = "REVIEWDOG"
+    report_type = "detailed"
+    scope = "linter"
+
+    def __init__(self, params=None):
+        super().__init__(params)
+        self.processor_config = None
+        if hasattr(self.master, "reviewdog_processor"):
+            print(f"RD: {self.master.reviewdog_processor}")
+            self.processor_config = self.master.reviewdog_processor
+
+    def manage_activation(self):
+        # Super-Linter legacy variables
+        output_format = config.get("OUTPUT_FORMAT", "")
+        if output_format.startswith("reviewdog"):
+            self.is_active = True
+        # Mega-Linter vars (false by default)
+        elif config.get("REVIEWDOG_REPORTER", "true") != "true":
+            self.is_active = False
+        else:
+            self.is_active = True
+
+    def produce_report(self):
+        # Files lines
+        text_report_lines = []
+        linter = self.master.linter_name.lower().replace("-", "_")
+        if self.master.cli_lint_mode == "file":
+            for file_result in self.master.files_lint_results:
+                file_nm = file_result["file"]
+                text_report_lines += [f"Linter log for {file_nm}"] + file_result["stdout"].splitlines()
+        # Bulk output as linter has run all project or files in one call
+        elif self.master.cli_lint_mode in ["project", "list_of_files"]:
+            workspace_nm = utils.normalize_log_string(self.master.workspace)
+            text_report_lines += self.master.stdout.splitlines()
+        # Complete lines
+        text_report_lines += self.master.complete_text_reporter_report(self)
+
+        # Convert to RDJsonl
+        if not self.processor_config:
+            logging.warning(
+                f"[Reviewdog Linter Reporter] Cannot generate Reviewdog report for {linter} as no processor is defined."
+                f"\n{vars(self.master)}"
+            )
+            return
+
+        processor = getattr(self, self.processor_config["class_name"])(self.processor_config["init_params"])
+        rdjsonl = processor.convert(text_report_lines)
+
+        # Write to file
+        text_report_sub_folder = config.get("REVIEWDOG_REPORTER_SUB_FOLDER", "reviewdog")
+        text_file_name = (
+            f"{self.report_folder}{os.path.sep}"
+            f"{text_report_sub_folder}{os.path.sep}"
+            f"{self.master.status.upper()}-{self.master.name}.log"
+        )
+        if not os.path.isdir(os.path.dirname(text_file_name)):
+            os.makedirs(os.path.dirname(text_file_name), exist_ok=True)
+        with open(text_file_name, "w", encoding="utf-8") as text_file:
+            text_file_content = "\n".join([json.dumps(line) for line in rdjsonl]) + "\n"
+            print(f"joined {linter} (length: {len(rdjsonl)}): {text_file_content}")
+            text_file.write(text_file_content)
+            logging.info(
+                f"[RDJsonl Reporter] Generated {self.name} report: {text_file_name}"
+            )
+
+    class RdjsonlConvertor:
+        def convert(self, outputlines):
+            return []
+
+    class Regex(RdjsonlConvertor):
+        def __init__(self, init_params) -> None:
+            self.regex = re.compile(init_params['regex'])
+            super().__init__()
+
+        def convert_line(self, line):
+            match = re.match(self.regex, line)
+            if not match:
+                logging.warning(f"Failed to extract data from line: {line}")
+                return []
+            return Rdjsonl(utils.normalize_log_string(match.group("path")), match.group("message"),
+                           start=Location(line=int(match.group("line")), column=int(match.group("column"))))
+
+        def convert(self, outputlines):
+            return [self.convert_line(line) for line in outputlines]
+
+    class UnifiedDiffs(RdjsonlConvertor):
+        def __init__(self, init_params) -> None:
+            self.path_regex = re.compile(init_params['file_header_regex'])
+            self.ignored_line_regexes = [re.compile(r) for r in init_params['ignored_line_regexes']]
+            super().__init__()
+
+        def suggestion(self, hunk):
+            target_lines = hunk.target
+            end_column = len(target_lines[-1])
+            text = "\n".join(target_lines)
+            return Suggestion(text, start=Location(hunk.source_start, 0), end=Location(hunk.source_start + hunk.source_length, end_column))
+
+        def process_udiff(self, udiff, path):
+            from unidiff import PatchSet
+            patches = PatchSet(udiff)
+            suggestions = [self.suggestion(hunk) for patch in patches for hunk in patch]
+            return [Rdjsonl(file=path, message=f"would reformat {path}",
+                            start=suggestion["range"]["start"], end=suggestion["range"]["end"],
+                            suggestions=suggestions)
+                    for suggestion in suggestions]
+
+        def convert(self, outputlines):
+            diffs = {}
+            current_file = None
+            for line in outputlines:
+                header_match = self.path_regex.match(line)
+                if header_match:
+                    current_file = header_match.group(1)
+                    diffs[current_file] = []
+                    continue
+                for ignore_line in self.ignored_line_regexes:
+                    ignore_line_match = ignore_line.match(line)
+                    if ignore_line_match:
+                        break
+                else:
+                    if current_file:
+                        diffs[current_file] += [line]
+            results = []
+            for file, udiff in diffs.items():
+                if udiff:
+                    results += self.process_udiff("\n".join(udiff), utils.normalize_log_string(file))
+            return results

megalinter/setup.py

@@ -27,6 +27,7 @@
         "mdx_truly_sane_lists",
         "beautifulsoup4",
         "giturlparse",
+        "unidiff",
     ],
     zip_safe=False,
 )

requirements.dev.txt

@@ -17,3 +17,4 @@ mdx_truly_sane_lists
 beautifulsoup4
 giturlparse
 json-schema-for-humans
+unidiff