From 05c757a84b1a5497606ddcf2d91f9f2a6c3c361b Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 1 Dec 2023 21:10:55 +0400 Subject: [PATCH 1/6] Replace short flags with long ones for clarity --- automations/python/workflows/get_folder_differences.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automations/python/workflows/get_folder_differences.py b/automations/python/workflows/get_folder_differences.py index 3eb79d7199..18e5506b34 100644 --- a/automations/python/workflows/get_folder_differences.py +++ b/automations/python/workflows/get_folder_differences.py @@ -49,7 +49,9 @@ def run_diff() -> str: command = " ".join( [ "diff", - "-qbr", + "--brief", + "--ignore-space-change", + "--recursive", *exclusion_args, str(OUTPUT_FOLDER), str(ORIGINAL_FOLDER / "_preview" / str(PR_NUMBER)), From 469019f781d9cb324de8a52c9e8a0c789e325962 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 1 Dec 2023 21:12:23 +0400 Subject: [PATCH 2/6] Emphasise that the current transform only applies to files --- .../python/workflows/get_folder_differences.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/automations/python/workflows/get_folder_differences.py b/automations/python/workflows/get_folder_differences.py index 18e5506b34..9db40add79 100644 --- a/automations/python/workflows/get_folder_differences.py +++ b/automations/python/workflows/get_folder_differences.py @@ -68,11 +68,15 @@ def run_diff() -> str: return completed.stdout -def convert_path_to_url(path: str) -> str: +def convert_file_path_to_url(path: str) -> str: """ - Convert a path to a URL that can be used to view the file on the docs + Convert a file path to a URL that can be used to view the file on the docs preview site. + + :param path: the file path to transform + :return: the transformed file path """ + # Remove the piece of the path before _preview path = "/_preview/" + path.split("/_preview/")[1] # Remove the _sources subfolder @@ -96,7 +100,7 @@ def process_diff(diff_output: str) -> tuple[list[str], list[str]]: if line.startswith("Files"): # e.g.: Files /tmp/gh-pages-for-diff/_sources/meta/index.md.txt and /tmp/gh-pages/_preview/2647/_sources/meta/index.md.txt differ # noqa: E501 updated = line.split()[3] - converted = convert_path_to_url(updated) + converted = convert_file_path_to_url(updated) if converted.endswith("html"): changed.append(converted) elif line.startswith("Only in"): @@ -104,7 +108,7 @@ def process_diff(diff_output: str) -> tuple[list[str], list[str]]: continue # e.g. Only in /tmp/gh-pages/_preview/2647/_sources/meta: examplefile.md.txt added = line.replace(": ", "/").split()[2] - converted = convert_path_to_url(added) + converted = convert_file_path_to_url(added) if converted.endswith("html"): new.append(converted) return changed, new From e1eb8f333ef3b8eb247716f37091baeddcef41f4 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 1 Dec 2023 21:17:52 +0400 Subject: [PATCH 3/6] Expand transformation to entire directories --- .../workflows/get_folder_differences.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/automations/python/workflows/get_folder_differences.py b/automations/python/workflows/get_folder_differences.py index 9db40add79..66fd46022a 100644 --- a/automations/python/workflows/get_folder_differences.py +++ b/automations/python/workflows/get_folder_differences.py @@ -87,6 +87,24 @@ def convert_file_path_to_url(path: str) -> str: return f"{BASE_URL}{path}" +def convert_file_or_dir_path_to_url(path: str) -> list[str]: + """ + Convert a file or directory path to URLs. + + This function applies the transformation ``convert_file_path_to_url`` to the + given file or every descendant file in the given directory. + + :param path: the file or directory path to transform + :return: the list of transformed file paths + """ + + if (path_obj := Path(path)).is_dir(): + paths = [str(item) for item in path_obj.rglob("*") if item.is_file()] + else: + paths = [path] + return list(map(convert_file_path_to_url, paths)) + + def process_diff(diff_output: str) -> tuple[list[str], list[str]]: """ Parse the output of the `diff` utility and create two lists of files: @@ -107,10 +125,10 @@ def process_diff(diff_output: str) -> tuple[list[str], list[str]]: if PR_NUMBER not in line: continue # e.g. Only in /tmp/gh-pages/_preview/2647/_sources/meta: examplefile.md.txt + # Note that the new addition could be a directory. added = line.replace(": ", "/").split()[2] - converted = convert_file_path_to_url(added) - if converted.endswith("html"): - new.append(converted) + converted = convert_file_or_dir_path_to_url(added) + new.extend(item for item in converted if item.endswith("html")) return changed, new From 770dfc8fb07c407ef65c7e0c8aa4e76a1902cd1c Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 1 Dec 2023 21:27:21 +0400 Subject: [PATCH 4/6] Add some test directories and files --- .../automations/reference/deleteme/deleteme/file_b.md | 3 +++ .../automations/reference/deleteme/deleteme/index.md | 9 +++++++++ documentation/automations/reference/deleteme/file_a.md | 3 +++ documentation/automations/reference/deleteme/index.md | 10 ++++++++++ documentation/automations/reference/index.md | 1 + 5 files changed, 26 insertions(+) create mode 100644 documentation/automations/reference/deleteme/deleteme/file_b.md create mode 100644 documentation/automations/reference/deleteme/deleteme/index.md create mode 100644 documentation/automations/reference/deleteme/file_a.md create mode 100644 documentation/automations/reference/deleteme/index.md diff --git a/documentation/automations/reference/deleteme/deleteme/file_b.md b/documentation/automations/reference/deleteme/deleteme/file_b.md new file mode 100644 index 0000000000..5bf3ad109e --- /dev/null +++ b/documentation/automations/reference/deleteme/deleteme/file_b.md @@ -0,0 +1,3 @@ +# File B + +I am a test file, delete me! diff --git a/documentation/automations/reference/deleteme/deleteme/index.md b/documentation/automations/reference/deleteme/deleteme/index.md new file mode 100644 index 0000000000..3f8137f1b0 --- /dev/null +++ b/documentation/automations/reference/deleteme/deleteme/index.md @@ -0,0 +1,9 @@ +# Deleteme (inner) + +I am a test directory, delete me! + +```{toctree} +:titlesonly: + +file_b +``` diff --git a/documentation/automations/reference/deleteme/file_a.md b/documentation/automations/reference/deleteme/file_a.md new file mode 100644 index 0000000000..6e36403e48 --- /dev/null +++ b/documentation/automations/reference/deleteme/file_a.md @@ -0,0 +1,3 @@ +# File A + +I am a test file, delete me! diff --git a/documentation/automations/reference/deleteme/index.md b/documentation/automations/reference/deleteme/index.md new file mode 100644 index 0000000000..0a13cd9741 --- /dev/null +++ b/documentation/automations/reference/deleteme/index.md @@ -0,0 +1,10 @@ +# Deleteme (outer) + +I am a test directory, delete me! + +```{toctree} +:titlesonly: + +file_a +deleteme/index +``` diff --git a/documentation/automations/reference/index.md b/documentation/automations/reference/index.md index 5edf29987f..750a7f12ec 100644 --- a/documentation/automations/reference/index.md +++ b/documentation/automations/reference/index.md @@ -5,4 +5,5 @@ python_scripts project_automations +deleteme/index ``` From 7c8dd41bb9220af8061e1ccae132c868c56da661 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 1 Dec 2023 21:56:23 +0400 Subject: [PATCH 5/6] Replicate exclusion rules for directories --- automations/python/workflows/get_folder_differences.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/automations/python/workflows/get_folder_differences.py b/automations/python/workflows/get_folder_differences.py index 66fd46022a..a45053fa1a 100644 --- a/automations/python/workflows/get_folder_differences.py +++ b/automations/python/workflows/get_folder_differences.py @@ -99,7 +99,12 @@ def convert_file_or_dir_path_to_url(path: str) -> list[str]: """ if (path_obj := Path(path)).is_dir(): - paths = [str(item) for item in path_obj.rglob("*") if item.is_file()] + paths = [ + str(item) + for item in path_obj.rglob("*") + if item.is_file() + and not any(item.match(exclusion) for exclusion in EXCLUSIONS) + ] else: paths = [path] return list(map(convert_file_path_to_url, paths)) From 3fc38023fd383d78876067119348c3261b99ebe9 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 1 Dec 2023 22:15:16 +0400 Subject: [PATCH 6/6] Undo all changes to the documentation --- .../automations/reference/deleteme/deleteme/file_b.md | 3 --- .../automations/reference/deleteme/deleteme/index.md | 9 --------- documentation/automations/reference/deleteme/file_a.md | 3 --- documentation/automations/reference/deleteme/index.md | 10 ---------- documentation/automations/reference/index.md | 1 - 5 files changed, 26 deletions(-) delete mode 100644 documentation/automations/reference/deleteme/deleteme/file_b.md delete mode 100644 documentation/automations/reference/deleteme/deleteme/index.md delete mode 100644 documentation/automations/reference/deleteme/file_a.md delete mode 100644 documentation/automations/reference/deleteme/index.md diff --git a/documentation/automations/reference/deleteme/deleteme/file_b.md b/documentation/automations/reference/deleteme/deleteme/file_b.md deleted file mode 100644 index 5bf3ad109e..0000000000 --- a/documentation/automations/reference/deleteme/deleteme/file_b.md +++ /dev/null @@ -1,3 +0,0 @@ -# File B - -I am a test file, delete me! diff --git a/documentation/automations/reference/deleteme/deleteme/index.md b/documentation/automations/reference/deleteme/deleteme/index.md deleted file mode 100644 index 3f8137f1b0..0000000000 --- a/documentation/automations/reference/deleteme/deleteme/index.md +++ /dev/null @@ -1,9 +0,0 @@ -# Deleteme (inner) - -I am a test directory, delete me! - -```{toctree} -:titlesonly: - -file_b -``` diff --git a/documentation/automations/reference/deleteme/file_a.md b/documentation/automations/reference/deleteme/file_a.md deleted file mode 100644 index 6e36403e48..0000000000 --- a/documentation/automations/reference/deleteme/file_a.md +++ /dev/null @@ -1,3 +0,0 @@ -# File A - -I am a test file, delete me! diff --git a/documentation/automations/reference/deleteme/index.md b/documentation/automations/reference/deleteme/index.md deleted file mode 100644 index 0a13cd9741..0000000000 --- a/documentation/automations/reference/deleteme/index.md +++ /dev/null @@ -1,10 +0,0 @@ -# Deleteme (outer) - -I am a test directory, delete me! - -```{toctree} -:titlesonly: - -file_a -deleteme/index -``` diff --git a/documentation/automations/reference/index.md b/documentation/automations/reference/index.md index 750a7f12ec..5edf29987f 100644 --- a/documentation/automations/reference/index.md +++ b/documentation/automations/reference/index.md @@ -5,5 +5,4 @@ python_scripts project_automations -deleteme/index ```