Refactoring/800 remove pretty prints and .lint.txt

.github/workflows/checks.yml

@@ -103,7 +103,6 @@ jobs:
         with:
           name: lint-python${{ matrix.python-versions }}
           path: |
-            .lint.txt
             .lint.json
           include-hidden-files: true
 

.gitignore

@@ -1,5 +1,4 @@
 .lint.json
-.lint.txt
 .security.json
 
 odbcconfig/odbcinst.ini

doc/changes/unreleased.md

@@ -1,3 +1,7 @@
 # Unreleased
 
 ## Summary
+
+## Refactoring
+
+* #800: Removed tbx security pretty-print, tbx lint pretty-print, and creation of .lint.txt, as superseded by Sonar and .lint.json usage

doc/user_guide/features/metrics/collecting_metrics.rst

@@ -61,8 +61,8 @@ sessions collect the artifacts from various jobs:
 |                          |   (unit, integration ...)                                |
 |                          | * Copies downloaded artifacts to their parent directory  |
 +--------------------------+----------------------------------------------------------+
-| ``artifacts:validate``   | * Verifies that the ``.lint.json``, ``.lint.txt``,       |
-|                          |   ``.security.json``, and ``.coverage`` are present      |
+| ``artifacts:validate``   | * Verifies that the ``.lint.json``, ``.security.json``,  |
+|                          |   and ``.coverage`` are present                          |
 |                          | * Checks that each file contains the expected attributes |
 |                          |   for that file type                                     |
 +--------------------------+----------------------------------------------------------+

exasol/toolbox/nox/_artifacts.py

@@ -1,6 +1,5 @@
 import json
 import os
-import re
 import shutil
 import sqlite3
 import subprocess  # nosec
@@ -17,10 +16,9 @@
 COVERAGE_DB = ".coverage"
 COVERAGE_XML = "ci-coverage.xml"
 LINT_JSON = ".lint.json"
-LINT_TXT = ".lint.txt"
 SECURITY_JSON = ".security.json"
 
-ALL_LINT_FILES = {COVERAGE_DB, LINT_JSON, LINT_TXT, SECURITY_JSON}
+ALL_LINT_FILES = {COVERAGE_DB, LINT_JSON, SECURITY_JSON}
 COVERAGE_TABLES = {"coverage_schema", "meta", "file", "line_bits"}
 LINT_JSON_ATTRIBUTES = {
     "type",
@@ -48,7 +46,6 @@ def check_artifacts(session: Session) -> None:
         sys.exit(1)
 
     all_is_valid_checks = [
-        _is_valid_lint_txt(Path(PROJECT_CONFIG.root_path, LINT_TXT)),
         _is_valid_lint_json(Path(PROJECT_CONFIG.root_path, LINT_JSON)),
         _is_valid_security_json(Path(PROJECT_CONFIG.root_path, SECURITY_JSON)),
         _is_valid_coverage(Path(PROJECT_CONFIG.root_path, COVERAGE_DB)),
@@ -62,16 +59,6 @@ def _handle_validation_error(file: Path, message: str) -> bool:
     return False
 
 
-def _is_valid_lint_txt(file: Path) -> bool:
-    content = file.read_text()
-    expr = re.compile(r"^Your code has been rated at (\d+.\d+)/.*", re.MULTILINE)
-    matches = expr.search(content)
-    if not matches:
-        _handle_validation_error(file, "Could not find a rating")
-        return False
-    return True
-
-
 def _is_valid_lint_json(file: Path) -> bool:
     try:
         content = file.read_text()
@@ -147,7 +134,6 @@ def copy_artifacts(session: Session) -> None:
         artifact_dir,
         artifact_dir.parent,
         [
-            f"lint{suffix}/{LINT_TXT}",
             f"lint{suffix}/{LINT_JSON}",
             f"security{suffix}/{SECURITY_JSON}",
         ],

exasol/toolbox/nox/_lint.py

@@ -17,12 +17,11 @@
 
 def _pylint(session: Session, files: Iterable[str]) -> None:
     json_file = PROJECT_CONFIG.root_path / ".lint.json"
-    txt_file = PROJECT_CONFIG.root_path / ".lint.txt"
 
     session.run(
         "pylint",
         "--output-format",
-        f"colorized,json:{json_file},text:{txt_file}",
+        f"colorized,json:{json_file}",
         *files,
     )
 

exasol/toolbox/templates/github/workflows/checks.yml

@@ -103,7 +103,6 @@ jobs:
         with:
           name: lint-python${{ matrix.python-versions }}
           path: |
-            .lint.txt
             .lint.json
           include-hidden-files: true
 

exasol/toolbox/tools/security.py

@@ -17,7 +17,6 @@
 from enum import Enum
 from functools import partial
 from inspect import cleandoc
-from pathlib import Path
 
 import typer
 
@@ -168,61 +167,6 @@ def from_pip_audit(report: str) -> Iterable[Issue]:
             )
 
 
-@dataclass(frozen=True)
-class SecurityIssue:
-    file_name: str
-    line: int
-    column: int
-    cwe: str
-    test_id: str
-    description: str
-    references: tuple
-
-
-def from_json(report_str: str, prefix: Path) -> Iterable[SecurityIssue]:
-    report = json.loads(report_str)
-    issues = report.get("results", {})
-    for issue in issues:
-        references = []
-        if issue["more_info"]:
-            references.append(issue["more_info"])
-        if issue.get("issue_cwe", {}).get("link", None):
-            references.append(issue["issue_cwe"]["link"])
-        yield SecurityIssue(
-            file_name=issue["filename"].replace(str(prefix) + "/", ""),
-            line=issue["line_number"],
-            column=issue["col_offset"],
-            cwe=str(issue["issue_cwe"].get("id", "")),
-            test_id=issue["test_id"],
-            description=issue["issue_text"],
-            references=tuple(references),
-        )
-
-
-def issues_to_markdown(issues: Iterable[SecurityIssue]) -> str:
-    template = cleandoc("""
-        {header}{rows}
-    """)
-
-    def _header():
-        header = "# Security\n\n"
-        header += "|File|line/<br>column|Cwe|Test ID|Details|\n"
-        header += "|---|:-:|:-:|:-:|---|\n"
-        return header
-
-    def _row(issue):
-        row = "|" + issue.file_name + "|"
-        row += f"line: {issue.line}<br>column: {issue.column}|"
-        row += issue.cwe + "|"
-        row += issue.test_id + "|"
-        for element in issue.references:
-            row += element + " ,<br>"
-        row = row[:-5] + "|"
-        return row
-
-    return template.format(header=_header(), rows="\n".join(_row(i) for i in issues))
-
-
 def security_issue_title(issue: Issue) -> str:
     return f"{issue.cve}: {issue.coordinates}"
 
@@ -384,23 +328,6 @@ def create(
         stdout(format_jsonl(issue_url, issue))
 
 
-class PPrintFormats(str, Enum):
-    markdown = "markdown"
-
-
-@CLI.command(name="pretty-print")
-def json_issue_to_markdown(
-    json_file: typer.FileText = typer.Argument(
-        mode="r", help="json file with issues to convert"
-    ),
-    path: Path = typer.Argument(default=Path("."), help="path to project root"),
-) -> None:
-    content = json_file.read()
-    issues = from_json(content, path.absolute())
-    issues = sorted(issues, key=lambda i: (i.file_name, i.cwe, i.test_id))
-    print(issues_to_markdown(issues))
-
-
 def format_jsonl(issue_url: str, issue: Issue) -> str:
     issue_json = asdict(issue)
     issue_json["issue_url"] = issue_url.strip()

exasol/toolbox/tools/tbx.py

@@ -2,14 +2,12 @@
 
 from exasol.toolbox.tools import (
     issue,
-    lint,
     security,
 )
 
 CLI = typer.Typer()
 CLI.add_typer(security.CLI, name="security", help="Security related helpers")
 CLI.add_typer(issue.CLI, name="issue", help="Manage issue templates")
-CLI.add_typer(lint.CLI, name="lint", help="linting related helpers")
 
 if __name__ == "__main__":
     CLI()

project-template/{{cookiecutter.repo_name}}/.gitignore

@@ -148,6 +148,5 @@ itde/
 
 # PTB
 /.lint.json
-/.lint.txt
 /.security.json
 /.sonar

test/integration/tools/security_integration_test.py

@@ -1,8 +1,6 @@
-import json
 from unittest.mock import patch
 
 from exasol.toolbox.tools.security import (
-    CLI,
     CVE_CLI,
     Filter,
     Format,
@@ -132,43 +130,3 @@ def test_works_as_expected_with_mocked_create_security_issue(
 
         assert result.exit_code == 0
         assert result.output.strip() == sample_maven_vulnerabilities.create_issues_json
-
-
-class TestJsonIssueToMarkdown:
-    @staticmethod
-    def test_with_filled_file(cli_runner, tmp_path):
-        json_path = tmp_path / "test.json"
-        json_path.write_text(json.dumps(JSON_RESULTS))
-
-        result = cli_runner.invoke(CLI, ["pretty-print", str(json_path)])
-
-        assert result.exit_code == 0
-        assert result.output.strip() == (
-            "# Security\n\n"
-            "|File|line/<br>column|Cwe|Test ID|Details|\n"
-            "|---|:-:|:-:|:-:|---|\n"
-            "|exasol/toolbox/sphinx/multiversion/git.py|line: 160<br>column: "
-            "12|22|B202|https://bandit.readthedocs.io/en/1.7.10/plugins/b202_tarfile_unsafe_members.html "
-            ",<br>https://cwe.mitre.org/data/definitions/22.html |\n"
-            "|exasol/toolbox/sphinx/multiversion/git.py|line: 157<br>column: "
-            "8|78|B603|https://bandit.readthedocs.io/en/1.7.10/plugins/b603_subprocess_without_shell_equals_true.html "
-            ",<br>https://cwe.mitre.org/data/definitions/78.html |\n"
-            "|exasol/toolbox/sphinx/multiversion/main.py|line: 556<br>column: "
-            "16|78|B602|https://bandit.readthedocs.io/en/1.7.10/plugins/b602_subprocess_popen_with_shell_equals_true.html "
-            ",<br>https://cwe.mitre.org/data/definitions/78.html |"
-        )
-
-    @staticmethod
-    def test_with_empty_file(cli_runner, tmp_path):
-        contents = {"result": []}
-        json_path = tmp_path / "test.json"
-        json_path.write_text(json.dumps(contents))
-
-        result = cli_runner.invoke(CLI, ["pretty-print", str(json_path)])
-
-        assert result.exit_code == 0
-        assert result.output.strip() == (
-            "# Security\n\n"
-            "|File|line/<br>column|Cwe|Test ID|Details|\n"
-            "|---|:-:|:-:|:-:|---|"
-        )