check_dead_links.py

# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import sys


def get_repo_root(file_path):
    """通过给定文件路径找到仓库根目录"""
    current_dir = os.path.dirname(file_path)
    while not os.path.exists(os.path.join(current_dir, ".git")):
        parent_dir = os.path.abspath(os.path.join(current_dir, os.pardir))
        if parent_dir == current_dir:
            raise FileNotFoundError("Could not find .git directory")
        current_dir = parent_dir
    return current_dir


def find_dead_links(directory):
    # 正则表达式，用于匹配Markdown和reStructuredText中的链接
    markdown_link_pattern = r"\[([^\[\]]+)\]\(([^)]+)\)"  # 修改正则表达式以捕获链接文本
    rst_link_pattern = r"``([^`]+) <([^>]+)>`_"  # reStructuredText链接
    dead_links = []

    for root, dirs, files in os.walk(directory):
        if "third_party" in root:
            continue
        for file in files:
            if file.endswith((".md", ".rst")):
                file_path = os.path.join(root, file)
                try:
                    with open(file_path, "r", encoding="utf-8") as f:
                        content = f.read()

                        # 查找Markdown链接
                        markdown_matches = re.findall(markdown_link_pattern, content)
                        for link_text, match in markdown_matches:
                            if match.startswith(("http:", "https:")):
                                # 忽略外部链接
                                continue
                            elif "#" in match:
                                # 这是一个锚点链接，忽略文件系统检查
                                continue
                            abs_path = os.path.abspath(os.path.join(os.path.dirname(file_path), match))
                            if not os.path.exists(abs_path):
                                dead_links.append((file_path, link_text, "Markdown Link: " + abs_path))

                        # 查找reStructuredText链接
                        rst_matches = re.findall(rst_link_pattern, content)
                        for text, url in rst_matches:
                            if not url.startswith(("http:", "https:")):
                                abs_path = os.path.abspath(os.path.join(os.path.dirname(file_path), url))
                                if not os.path.exists(abs_path):
                                    dead_links.append((file_path, text, "reStructuredText Link: " + abs_path))

                except Exception as e:
                    print(f"Error reading {file_path}: {e}")

    return dead_links


def create_symlinks(root_dir, src_dir, tgt_dir, file_extension=".md"):
    """
    Create corresponding folders in the tgt directory based on the src directory,
    and create relative path symlinks for files of a specific type.
    Also check if existing files in tgt have corresponding files in src, otherwise print a warning.

    :param src_dir: Path to the source directory
    :param tgt_dir: Path to the target directory
    :param file_extension: File extension for which symlinks need to be created, default is ".md"
    """
    tgt_dir = os.path.join(root_dir, tgt_dir)
    src_dir = os.path.join(root_dir, src_dir)

    # List all existing files in the tgt directory (including files in subdirectories)
    existing_tgt_files = set()
    for root, dirs, files in os.walk(tgt_dir):
        for file in files:
            existing_tgt_files.add(os.path.relpath(os.path.join(root, file), tgt_dir))

    # Ensure the target directory exists
    os.makedirs(tgt_dir, exist_ok=True)

    count = 0

    # Iterate over all files and folders in the source directory
    for root, dirs, files in os.walk(src_dir):
        # Create corresponding folder structure in the target directory
        relative_path = os.path.relpath(root, src_dir)
        tgt_path = os.path.join(tgt_dir, relative_path)

        # Create symlinks for files of a specific type
        for file in files:
            if file.endswith(file_extension):
                src_file_path = os.path.join(root, file)
                relative_src_file_path = os.path.relpath(src_file_path, tgt_path)
                tgt_file_path = os.path.join(tgt_path, file)

                os.makedirs(tgt_path, exist_ok=True)

                # If the target file already exists and is a symlink, delete it first
                if os.path.exists(tgt_file_path) and os.path.islink(tgt_file_path):
                    existing_link_target = os.readlink(tgt_file_path)
                    if existing_link_target != relative_src_file_path:
                        os.unlink(tgt_file_path)
                        # Create the symlink
                        os.symlink(relative_src_file_path, tgt_file_path)
                        count += 1

                elif not os.path.exists(tgt_file_path):
                    os.symlink(relative_src_file_path, tgt_file_path)
                    count += 1
                else:
                    print(f"File already exists: {tgt_file_path}. Please remove it from {tgt_dir} and try again.")
                    sys.exit(1)

                # Remove this processed file from the existing tgt files
                existing_tgt_files.discard(os.path.relpath(tgt_file_path, tgt_dir))

    # Check for remaining files in tgt (i.e., files that exist in tgt but not found in src)
    for file in existing_tgt_files:
        print(f"Warning: File exists in {tgt_dir} but not found in {src_dir}: {file}")

    return count


def process_file(file_path):
    # Default synchronization of the 'llm' and 'docs/zh/llm' folders
    count = create_symlinks(file_path, "llm", "docs/zh/llm", file_extension=".md")
    if count > 0:
        print("New files were added to docs/zh/llm. Please check them.")
        sys.exit(1)

    dead_links = find_dead_links(file_path)
    if len(dead_links) > 0:
        print("Dead links found in", file_path)
        for link in dead_links:
            print("file path:", link[0], "- link text:", link[1], "- deal link:", link[2])
        print("Please check the above dead links and fix them.")
        sys.exit(1)


if __name__ == "__main__":
    if len(sys.argv) > 1:
        repo_root = get_repo_root(os.path.realpath(sys.argv[1]))
        process_file(repo_root)