Add reporter for outputing Reviewdog JSONL format

This reporter will convert linter output to a standardised format that can
easily be read by reviewdog.

I have decided to use the JSONL format over the JSON format as it will be
simpler to join together mutltiple files before feeding into review dog.

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 Note: Can be used with `nvuillam/mega-linter@insiders` in your GitHub Action mega-linter.yml file, or with `nvuillam/mega-linter@latest` docker image
 
+- Add Reviewdog JSONL Reporter
+
 - Linter versions upgrades
   - [flake8](https://flake8.pycqa.org) from 3.8.4 to **3.9.0** on 2021-03-15
 <!-- linter-versions-end -->

megalinter/reporters/ReviewdogJsonlReporter.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""
+Reviewdog JSONL reporter
+See https://github.com/reviewdog/reviewdog/tree/master/proto/rdf#rdjsonl for details
+"""
+import json
+import logging
+import os
+import re
+
+from megalinter import Reporter, config, utils
+
+
+class Location(dict):
+    def __init__(self, line, column) -> None:
+        super().__init__(line=line, column=column)
+
+
+class Suggestion(dict):
+    def __init__(self, suggestion_text, start, end) -> None:
+        super().__init__(text=suggestion_text, range={"start": start, "end": end})
+
+
+class Rdjsonl(dict):
+    def __init__(self, file, message, start, end=None, severity="ERROR", suggestions=None) -> None:
+        if suggestions is None:
+            suggestions = []
+        location = {"range": {"start": start, "end": end}, "path": file}
+        super().__init__(message=message, location=location, severity=severity, suggestions=suggestions)
+
+
+class ReviewdogJsonlReporter(Reporter):
+    name = "REVIEWDOG"
+    report_type = "detailed"
+    scope = "linter"
+
+    def __init__(self, params=None):
+        self.convertors = {c.linter: c() for c in self.RdjsonlConvertor.__subclasses__()}
+        super().__init__(params)
+
+    def manage_activation(self):
+        # Super-Linter legacy variables
+        output_format = config.get("OUTPUT_FORMAT", "")
+        if output_format.startswith("reviewdog"):
+            self.is_active = True
+        # Mega-Linter vars (false by default)
+        elif config.get("REVIEWDOG_REPORTER", "false") != "true":
+            self.is_active = False
+        else:
+            self.is_active = True
+
+    def produce_report(self):
+        # Files lines
+        text_report_lines = []
+        linter = self.master.linter_name.lower().replace("-", "_")
+        if self.master.cli_lint_mode == "file":
+            for file_result in self.master.files_lint_results:
+                file_nm = file_result["file"]
+                text_report_lines += [f"Linter log for {file_nm}"] + file_result["stdout"].splitlines()
+        # Bulk output as linter has run all project or files in one call
+        elif self.master.cli_lint_mode in ["project", "list_of_files"]:
+            workspace_nm = utils.normalize_log_string(self.master.workspace)
+            text_report_lines += self.master.stdout.splitlines()
+        # Complete lines
+        text_report_lines += self.master.complete_text_reporter_report(self)
+
+        # Convert to RDJsonl
+        convertor = self.convertors.get(linter, self.RdjsonlConvertor())
+        rdjsonl = convertor.convert(text_report_lines)
+
+        # Write to file
+        text_report_sub_folder = config.get("REVIEWDOG_REPORTER_SUB_FOLDER", "reviewdog")
+        text_file_name = (
+            f"{self.report_folder}{os.path.sep}"
+            f"{text_report_sub_folder}{os.path.sep}"
+            f"{self.master.status.upper()}-{self.master.name}.log"
+        )
+        if not os.path.isdir(os.path.dirname(text_file_name)):
+            os.makedirs(os.path.dirname(text_file_name), exist_ok=True)
+        with open(text_file_name, "w", encoding="utf-8") as text_file:
+            text_file_content = "\n".join([json.dumps(line) for line in rdjsonl]) + "\n"
+            print(f"joined {linter} (length: {len(text_report_lines)}): {text_file_content}")
+            text_file.write(text_file_content)
+            logging.info(
+                f"[RDJsonl Reporter] Generated {self.name} report: {text_file_name}"
+            )
+
+    class RdjsonlConvertor:
+        linter = "default"
+
+        def convert(self, outputlines):
+            return []
+
+    class PythonFlake8(RdjsonlConvertor):
+        linter = "flake8"
+
+        def convert_line(self, line):
+            components = line.split(":")
+            return Rdjsonl(utils.normalize_log_string(components[0]), components[3],
+                           start=Location(line=int(components[1]), column=int(components[2])))
+
+        def convert(self, outputlines):
+            return [self.convert_line(line) for line in outputlines]
+
+    class PythonBlack(RdjsonlConvertor):
+        linter = "black"
+
+        def convert_line(self, line):
+            print(f"blackLine: {line}")
+            return None
+
+        def suggestion(self, hunk):
+            target_lines = hunk.target
+            end_column = len(target_lines[-1])
+            text = "\n".join(target_lines)
+            return Suggestion(text, start=Location(hunk.source_start, 0), end=Location(hunk.source_start + hunk.source_length, end_column))
+
+        def process_udiff(self, udiff, path):
+            from unidiff import PatchSet
+            patches = PatchSet(udiff)
+            suggestions = [self.suggestion(hunk) for patch in patches for hunk in patch]
+            return [Rdjsonl(file=path, message=f"would reformat {path}",
+                            start=suggestion["range"]["start"], end=suggestion["range"]["end"],
+                            suggestions=suggestions)
+                    for suggestion in suggestions]
+
+        def convert(self, outputlines):
+            diffs = {}
+            current_file = None
+            for line in outputlines:
+                if line.startswith("Linter log for "):
+                    current_file = re.match(r"^Linter log for (.+)$", line).group(1)
+                    diffs[current_file] = []
+                elif line.startswith("All done!") or line.endswith("file would be left unchanged."):
+                    pass  # we can skip these lines as they are part of the output for a file that has no errors
+                elif line.startswith("Oh no!") or line.endswith("file would be reformatted."):
+                    pass  # we can skip these lines as we already know we are processing lines for files with errors
+                elif current_file:
+                    diffs[current_file] += [line]
+            results = []
+            for file, udiff in diffs.items():
+                if udiff:
+                    results += self.process_udiff("\n".join(udiff), utils.normalize_log_string(file))
+            return results

megalinter/setup.py

@@ -27,6 +27,7 @@
         "mdx_truly_sane_lists",
         "beautifulsoup4",
         "giturlparse",
+        "unidiff",
     ],
     zip_safe=False,
 )

requirements.dev.txt

@@ -17,3 +17,4 @@ mdx_truly_sane_lists
 beautifulsoup4
 giturlparse
 json-schema-for-humans
+unidiff