diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index b78c99efb9be3..8d02baa05f489 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -94,6 +94,10 @@ COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/cla COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/clang-tidy-diff.py # Install dependencies for 'pr-code-lint.yml' job +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y python3-doc8 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* COPY llvm/utils/git/requirements_linting.txt requirements_linting.txt RUN pip install -r requirements_linting.txt --break-system-packages && \ rm requirements_linting.txt diff --git a/.github/workflows/pr-code-lint.yml b/.github/workflows/pr-code-lint.yml index 5444a29c22205..4f016b42d180c 100644 --- a/.github/workflows/pr-code-lint.yml +++ b/.github/workflows/pr-code-lint.yml @@ -30,7 +30,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 2 - + - name: Get changed files id: changed-files uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 @@ -39,14 +39,14 @@ jobs: skip_initial_fetch: true base_sha: 'HEAD~1' sha: 'HEAD' - + - name: Listed files env: CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} run: | echo "Changed files:" echo "$CHANGED_FILES" - + # TODO: create special mapping for 'codegen' targets, for now build predefined set # TODO: add entrypoint in 'compute_projects.py' that only adds a project and its direct dependencies - name: Configure and CodeGen @@ -71,12 +71,17 @@ jobs: -DLLVM_INCLUDE_TESTS=OFF \ -DCLANG_INCLUDE_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release - + ninja -C build \ clang-tablegen-targets \ genconfusable # for "ConfusableIdentifierCheck.h" - - name: Run code linter + - name: Install linter dependencies + run: | + pip install doc8 --break-system-packages + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Run linters env: GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} @@ -89,7 +94,7 @@ jobs: --end-rev HEAD \ --verbose \ --changed-files "$CHANGED_FILES" - + - name: Upload results uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: always() diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst index cb7ea415c54b2..6f9f0df8ba3d5 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst @@ -3,6 +3,7 @@ bugprone-unsafe-functions ========================= + Checks for functions that have safer, more secure replacements available, or are considered deprecated due to design flaws. The check heavily relies on the functions from the diff --git a/llvm/utils/git/code-lint-helper.py b/llvm/utils/git/code-lint-helper.py index 1232f3ab0d370..0821c6830deb2 100755 --- a/llvm/utils/git/code-lint-helper.py +++ b/llvm/utils/git/code-lint-helper.py @@ -34,82 +34,53 @@ class LintArgs: issue_number: int = 0 build_path: str = "build" clang_tidy_binary: str = "clang-tidy" + doc8_binary: str = "doc8" - def __init__(self, args: argparse.Namespace = None) -> None: + def __init__(self, args: argparse.Namespace) -> None: if not args is None: self.start_rev = args.start_rev self.end_rev = args.end_rev self.repo = args.repo self.token = args.token - self.changed_files = args.changed_files + if args.changed_files: + self.changed_files = args.changed_files.split(",") + else: + self.changed_files = [] self.issue_number = args.issue_number self.verbose = args.verbose self.build_path = args.build_path self.clang_tidy_binary = args.clang_tidy_binary + self.doc8_binary = args.doc8_binary -COMMENT_TAG = "" +class LintHelper: + COMMENT_TAG = "" + name: str + friendly_name: str + comment: dict = None + @property + def comment_tag(self) -> str: + return self.COMMENT_TAG.format(linter=self.name) -def get_instructions(cpp_files: List[str]) -> str: - files_str = " ".join(cpp_files) - return f""" -git diff -U0 origin/main...HEAD -- {files_str} | -python3 clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py \\ - -path build -p1 -quiet""" - - -def clean_clang_tidy_output(output: str) -> Optional[str]: - """ - - Remove 'Running clang-tidy in X threads...' line - - Remove 'N warnings generated.' line - - Strip leading workspace path from file paths - """ - if not output or output == "No relevant changes found.": - return None - - lines = output.split("\n") - cleaned_lines = [] - - for line in lines: - if line.startswith("Running clang-tidy in") or line.endswith("generated."): - continue - - # Remove everything up to rightmost "llvm-project/" for correct files names - idx = line.rfind("llvm-project/") - if idx != -1: - line = line[idx + len("llvm-project/") :] + @property + def instructions(self) -> str: + raise NotImplementedError() - cleaned_lines.append(line) + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + raise NotImplementedError() - if cleaned_lines: - return "\n".join(cleaned_lines) - return None + def run_linter_tool( + self, files_to_lint: List[str], args: LintArgs + ) -> Optional[str]: + raise NotImplementedError() - -# TODO: Add more rules when enabling other projects to use clang-tidy in CI. -def should_lint_file(filepath: str) -> bool: - return filepath.startswith("clang-tools-extra/clang-tidy/") - - -def filter_changed_files(changed_files: List[str]) -> List[str]: - filtered_files = [] - for filepath in changed_files: - _, ext = os.path.splitext(filepath) - if ext not in (".cpp", ".c", ".h", ".hpp", ".hxx", ".cxx"): - continue - if not should_lint_file(filepath): - continue - if os.path.exists(filepath): - filtered_files.append(filepath) - - return filtered_files - - -def create_comment_text(warning: str, cpp_files: List[str]) -> str: - instructions = get_instructions(cpp_files) - return f""" -:warning: C/C++ code linter clang-tidy found issues in your code. :warning: + def pr_comment_text_for_diff( + self, linter_output: str, files_to_lint: List[str], args: LintArgs + ) -> str: + instructions = self.instructions(files_to_lint, args) + return f""" +:warning: {self.friendly_name}, {self.name} found issues in your code. :warning:
@@ -124,133 +95,241 @@ def create_comment_text(warning: str, cpp_files: List[str]) -> str:
-View the output from clang-tidy here. +View the output from {self.name} here. ``` -{warning} +{linter_output} ```
""" + def find_comment(self, pr: any) -> any: + for comment in pr.as_issue().get_comments(): + if self.comment_tag in comment.body: + return comment + return None -def find_comment(pr: any) -> any: - for comment in pr.as_issue().get_comments(): - if COMMENT_TAG in comment.body: - return comment - return None + def update_pr(self, comment_text: str, args: LintArgs, create_new: bool) -> None: + import github + from github import IssueComment, PullRequest + repo = github.Github(args.token).get_repo(args.repo) + pr = repo.get_issue(args.issue_number).as_pull_request() -def create_comment( - comment_text: str, args: LintArgs, create_new: bool -) -> Optional[dict]: - import github + comment_text = self.comment_tag + "\n\n" + comment_text - repo = github.Github(args.token).get_repo(args.repo) - pr = repo.get_issue(args.issue_number).as_pull_request() + existing_comment = self.find_comment(pr) - comment_text = COMMENT_TAG + "\n\n" + comment_text + if create_new or existing_comment: + self.comment = {"body": comment_text} + if existing_comment: + self.comment["id"] = existing_comment.id - existing_comment = find_comment(pr) - comment = None - if create_new or existing_comment: - comment = {"body": comment_text} - if existing_comment: - comment["id"] = existing_comment.id - return comment + def run(self, args: LintArgs) -> bool: + files_to_lint = self.filter_changed_files(args.changed_files) + is_success = True + linter_output = None -def run_clang_tidy(changed_files: List[str], args: LintArgs) -> Optional[str]: - if not changed_files: - print("no c/c++ files found") - return None + if files_to_lint: + linter_output = self.run_linter_tool(files_to_lint, args) + if linter_output: + is_success = False - git_diff_cmd = [ - "git", - "diff", - "-U0", - f"{args.start_rev}...{args.end_rev}", - "--", - ] + changed_files - - diff_proc = subprocess.run( - git_diff_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=False, - ) + should_update_gh = args.token is not None and args.repo is not None - if diff_proc.returncode != 0: - print(f"Git diff failed: {diff_proc.stderr}") - return None + if is_success: + if should_update_gh: + comment_text = ( + ":white_check_mark: With the latest revision " + f"this PR passed the {self.friendly_name}." + ) + self.update_pr(comment_text, args, create_new=False) + return True + else: + if should_update_gh: + if linter_output: + comment_text = self.pr_comment_text_for_diff( + linter_output, files_to_lint, args + ) + self.update_pr(comment_text, args, create_new=True) + else: + comment_text = ( + f":warning: The {self.friendly_name} failed without printing " + "an output. Check the logs for output. :warning:" + ) + self.update_pr(comment_text, args, create_new=False) + else: + if linter_output: + print( + f"Warning: {self.friendly_name}, {self.name} detected " + "some issues with your code..." + ) + print(linter_output) + else: + print(f"Warning: {self.friendly_name}, {self.name} failed to run.") + return False + + +class ClangTidyLintHelper(LintHelper): + name = "clang-tidy" + friendly_name = "C/C++ code linter" + + def instructions(self, cpp_files: List[str], args: LintArgs) -> str: + files_str = " ".join(cpp_files) + return f""" +git diff -U0 origin/main...HEAD -- {files_str} | +python3 clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py \\ + -path {args.build_path} -p1 -quiet""" + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + clang_tidy_changed_files = [ + arg for arg in changed_files if "third-party" not in arg + ] + + filtered_files = [] + for filepath in clang_tidy_changed_files: + _, ext = os.path.splitext(filepath) + if ext not in (".cpp", ".c", ".h", ".hpp", ".hxx", ".cxx"): + continue + if not self._should_lint_file(filepath): + continue + if os.path.exists(filepath): + filtered_files.append(filepath) + return filtered_files + + def _should_lint_file(self, filepath: str) -> bool: + # TODO: Add more rules when enabling other projects to use clang-tidy in CI. + return filepath.startswith("clang-tools-extra/clang-tidy/") + + def run_linter_tool(self, cpp_files: List[str], args: LintArgs) -> Optional[str]: + if not cpp_files: + return None + + git_diff_cmd = [ + "git", + "diff", + "-U0", + f"{args.start_rev}...{args.end_rev}", + "--", + ] + cpp_files + + diff_proc = subprocess.run( + git_diff_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) - diff_content = diff_proc.stdout - if not diff_content.strip(): - print("No diff content found") - return None + if diff_proc.returncode != 0: + print(f"Git diff failed: {diff_proc.stderr}") + return "Git diff failed" - tidy_diff_cmd = [ - "clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py", - "-path", - args.build_path, - "-p1", - "-quiet", - ] + diff_content = diff_proc.stdout + if not diff_content.strip(): + return None - if args.verbose: - print(f"Running clang-tidy-diff: {' '.join(tidy_diff_cmd)}") - - proc = subprocess.run( - tidy_diff_cmd, - input=diff_content, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=False, - ) + tidy_diff_cmd = [ + "clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py", + "-path", + args.build_path, + "-p1", + "-quiet", + ] - return clean_clang_tidy_output(proc.stdout.strip()) + if args.verbose: + print(f"Running clang-tidy-diff: {' '.join(tidy_diff_cmd)}") + + proc = subprocess.run( + tidy_diff_cmd, + input=diff_content, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + clean_output = self._clean_clang_tidy_output(proc.stdout.strip()) + return clean_output -def run_linter(changed_files: List[str], args: LintArgs) -> tuple[bool, Optional[dict]]: - changed_files = [arg for arg in changed_files if "third-party" not in arg] + def _clean_clang_tidy_output(self, output: str) -> Optional[str]: + if not output or output == "No relevant changes found.": + return None - cpp_files = filter_changed_files(changed_files) + lines = output.split("\n") + cleaned_lines = [] - tidy_result = run_clang_tidy(cpp_files, args) - should_update_gh = args.token is not None and args.repo is not None + for line in lines: + if line.startswith("Running clang-tidy in") or line.endswith("generated."): + continue - comment = None - if tidy_result is None: - if should_update_gh: - comment_text = ( - ":white_check_mark: With the latest revision " - "this PR passed the C/C++ code linter." - ) - comment = create_comment(comment_text, args, create_new=False) - return True, comment - elif len(tidy_result) > 0: - if should_update_gh: - comment_text = create_comment_text(tidy_result, cpp_files) - comment = create_comment(comment_text, args, create_new=True) - else: - print( - "Warning: C/C++ code linter, clang-tidy detected " - "some issues with your code..." - ) - return False, comment - else: - # The linter failed but didn't output a result (e.g. some sort of - # infrastructure failure). - comment_text = ( - ":warning: The C/C++ code linter failed without printing " - "an output. Check the logs for output. :warning:" + idx = line.rfind("llvm-project/") + if idx != -1: + line = line[idx + len("llvm-project/") :] + + cleaned_lines.append(line) + + if cleaned_lines: + return "\n".join(cleaned_lines) + return None + + +class Doc8LintHelper(LintHelper): + name = "doc8" + friendly_name = "Documentation linter" + + def instructions(self, doc_files: List[str], args: LintArgs) -> str: + files_str = " ".join(doc_files) + return f"doc8 -q {files_str}" + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for filepath in changed_files: + _, ext = os.path.splitext(filepath) + if ext not in (".rst"): + continue + if not filepath.startswith("clang-tools-extra/docs/clang-tidy/checks/"): + continue + if os.path.exists(filepath): + filtered_files.append(filepath) + return filtered_files + + def run_linter_tool(self, doc_files: List[str], args: LintArgs) -> Optional[str]: + if not doc_files: + return None + + doc8_cmd = [args.doc8_binary, "-q"] + doc_files + + if args.verbose: + print(f"Running doc8: {' '.join(doc8_cmd)}") + + proc = subprocess.run( + doc8_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, ) - comment = create_comment(comment_text, args, create_new=False) - return False, comment + + if proc.returncode == 0: + return None + + output = proc.stdout.strip() + if output: + return output + + error_output = proc.stderr.strip() + if error_output: + return error_output + + return f"doc8 exited with return code {proc.returncode} but no output." + + +ALL_LINTERS = (ClangTidyLintHelper(), Doc8LintHelper()) if __name__ == "__main__": @@ -291,6 +370,12 @@ def run_linter(changed_files: List[str], args: LintArgs) -> tuple[bool, Optional default="clang-tidy", help="Path to clang-tidy binary", ) + parser.add_argument( + "--doc8-binary", + type=str, + default="doc8", + help="Path to doc8 binary", + ) parser.add_argument( "--verbose", action="store_true", default=True, help="Verbose output" ) @@ -298,32 +383,31 @@ def run_linter(changed_files: List[str], args: LintArgs) -> tuple[bool, Optional parsed_args = parser.parse_args() args = LintArgs(parsed_args) - changed_files = [] - if args.changed_files: - changed_files = args.changed_files.split(",") - if args.verbose: - print(f"got changed files: {changed_files}") + print("Running all linters.") - if args.verbose: - print("running linter clang-tidy") - - success, comment = run_linter(changed_files, args) + overall_success = True + all_comments = [] - if not success: + for linter in ALL_LINTERS: if args.verbose: - print("linter clang-tidy failed") + print(f"Running linter: {linter.name}") - # Write comments file if we have a comment - if comment: - if args.verbose: - print(f"linter clang-tidy has comment: {comment}") + linter_passed = linter.run(args) + if not linter_passed: + overall_success = False - with open("comments", "w") as f: - import json + if linter.comment: + all_comments.append(linter.comment) - json.dump([comment], f) + if len(all_comments): + import json - if not success: - print("error: some linters failed: clang-tidy") + with open("comments", "w") as f: + json.dump(all_comments, f) + + if not overall_success: + print("error: Some linters failed.") sys.exit(1) + else: + print("All linters passed.")