From 3f2589feb3a5fe79c5ca64b357d187bedada8ca9 Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 16:10:28 -0400 Subject: [PATCH 1/9] Add argument parser to copyright header script and integrate with pre-commit - Add --check and --fix arguments to check-copyright-header.py - --check: only checks files and reports missing headers, exits with code 1 if any found - --fix: automatically adds missing headers (default behavior) - Add copyright header check to .pre-commit-config.yaml - Script now properly handles all source files in python/, tests/, scripts/, and examples/ directories Signed-off-by: Yaoyao Ding --- .pre-commit-config.yaml | 18 ++ examples/blackwell_matmul/matmul_v0.py | 2 + examples/blackwell_matmul/matmul_v1.py | 2 + .../extensions/hidet/ir/primitives/swizzle.py | 14 ++ .../transforms/check_launch_configuration.py | 14 ++ scripts/lint/check-copyright-header.py | 179 ++++++++++++++++++ tests/instructions/test_print_tensor.py | 14 ++ 7 files changed, 243 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 scripts/lint/check-copyright-header.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..727fb2fc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files +- repo: local + hooks: + - id: check-copyright-header + name: Check copyright headers + entry: python scripts/lint/check-copyright-header.py --check + language: system + types: [python] + pass_filenames: false diff --git a/examples/blackwell_matmul/matmul_v0.py b/examples/blackwell_matmul/matmul_v0.py index 204a8227..bbfc2da0 100644 --- a/examples/blackwell_matmul/matmul_v0.py +++ b/examples/blackwell_matmul/matmul_v0.py @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 import os import pandas diff --git a/examples/blackwell_matmul/matmul_v1.py b/examples/blackwell_matmul/matmul_v1.py index f2280b98..86408fdd 100644 --- a/examples/blackwell_matmul/matmul_v1.py +++ b/examples/blackwell_matmul/matmul_v1.py @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 import os import pandas diff --git a/python/tilus/extensions/hidet/ir/primitives/swizzle.py b/python/tilus/extensions/hidet/ir/primitives/swizzle.py index 39736464..875e72d1 100644 --- a/python/tilus/extensions/hidet/ir/primitives/swizzle.py +++ b/python/tilus/extensions/hidet/ir/primitives/swizzle.py @@ -1,3 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import no_type_check from hidet.ir.dtypes import int32 diff --git a/python/tilus/extensions/hidet/transforms/check_launch_configuration.py b/python/tilus/extensions/hidet/transforms/check_launch_configuration.py index a3d2a3a4..76f7d7e0 100644 --- a/python/tilus/extensions/hidet/transforms/check_launch_configuration.py +++ b/python/tilus/extensions/hidet/transforms/check_launch_configuration.py @@ -1,3 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/scripts/lint/check-copyright-header.py b/scripts/lint/check-copyright-header.py new file mode 100644 index 00000000..cbb54166 --- /dev/null +++ b/scripts/lint/check-copyright-header.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import sys +from datetime import datetime + +current_year = datetime.now().year + +LICENSE_HEADER = """ +# SPDX-FileCopyrightText: Copyright (c) {year} NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""".format(year=current_year) + +SHORT_LICENSE_HEADER = """ +# SPDX-FileCopyrightText: Copyright (c) {year} NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +""".format(year=current_year) + +LICENSE_LINE = "# SPDX-License-Identifier: Apache-2.0\n" +TARGET_DIRS = [ + os.path.join(os.path.dirname(__file__), "..", "..", "python"), + os.path.join(os.path.dirname(__file__), "..", "..", "tests"), + os.path.join(os.path.dirname(__file__), "..", "..", "scripts"), +] +SHORT_LICENSE_TARGET_DIRS = [ + os.path.join(os.path.dirname(__file__), "..", "..", "examples"), +] + + +def add_license_to_file(filepath, filetype, use_short_header=False): + with open(filepath, "r", encoding="utf-8") as f: + lines = f.readlines() + + # Choose which header to use + header_to_use = SHORT_LICENSE_HEADER if use_short_header else LICENSE_HEADER + + # Find where to insert (after shebang if present, else at top) + insert_idx = 1 if lines and lines[0].startswith("#!") else 0 + + # Check first 15 lines for any existing license header + search_lines = lines[:15] + + # Check if any header already exists (with SPDX-FileCopyrightText) + has_any_header = any("SPDX-FileCopyrightText" in line for line in search_lines) + if has_any_header: + return False # Header already exists, don't modify + + # Check if only the short license line exists + has_license_line = any(LICENSE_LINE.strip() in line for line in search_lines) + + if has_license_line: + # Replace the short license line with the appropriate header + # Find and remove the existing short license line + for i, line in enumerate(lines): + if LICENSE_LINE.strip() in line: + lines.pop(i) + break + # Insert the appropriate header at the right position + header_lines = header_to_use.split("\n") + for i, header_line in enumerate(reversed(header_lines)): + if header_line: # Skip empty lines at the end + lines.insert(insert_idx, header_line + "\n") + else: + # No license header exists, add the appropriate header + header_lines = header_to_use.split("\n") + for i, header_line in enumerate(reversed(header_lines)): + if header_line: # Skip empty lines at the end + lines.insert(insert_idx, header_line + "\n") + + with open(filepath, "w", encoding="utf-8") as f: + f.writelines(lines) + return True + + +def process_directory(target_dirs, use_short_header=False, check_only=False): + total_files = 0 + need_update = 0 + updated = 0 + files_needing_update = [] + + for target_dir in target_dirs: + for root, _, files in os.walk(target_dir): + for file in files: + if file.endswith(".py") or file.endswith(".sh"): + total_files += 1 + filepath = os.path.join(root, file) + with open(filepath, "r", encoding="utf-8") as f: + lines = f.readlines() + + # Check first 15 lines for any existing license header + search_lines = lines[:15] + has_any_header = any("SPDX-FileCopyrightText" in line for line in search_lines) + + # File needs update if it has no header + if not has_any_header: + need_update += 1 + files_needing_update.append(filepath) + if not check_only: + if add_license_to_file( + filepath, filetype=file.split(".")[-1], use_short_header=use_short_header + ): + updated += 1 + + return total_files, need_update, updated, files_needing_update + + +def main(): + parser = argparse.ArgumentParser(description="Check and fix copyright headers in source files") + parser.add_argument("--check", action="store_true", + help="Only check if files have headers, don't modify them") + parser.add_argument("--fix", action="store_true", + help="Automatically add missing headers to files") + + args = parser.parse_args() + + # Default behavior is fix if neither check nor fix is specified + if not args.check and not args.fix: + args.fix = True + + check_only = args.check # Process directories with full license header + full_total, full_need_update, full_updated, full_files_needing_update = process_directory( + TARGET_DIRS, use_short_header=False, check_only=check_only) + + # Process directories with short license header + short_total, short_need_update, short_updated, short_files_needing_update = process_directory( + SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=check_only) + + # Combined totals + total_files = full_total + short_total + need_update = full_need_update + short_need_update + updated = full_updated + short_updated + all_files_needing_update = full_files_needing_update + short_files_needing_update + + if check_only: + if need_update > 0: + print(f"Found {need_update} files missing copyright headers:") + for filepath in all_files_needing_update: + print(f" {filepath}") + sys.exit(1) + else: + print(f"All {total_files} source files have copyright headers.") + sys.exit(0) + else: + print(f"Total source files found (.py/.sh): {total_files}") + print(f"Files with full header: {full_total} (need update: {full_need_update}, updated: {full_updated})") + print(f"Files with short header: {short_total} (need update: {short_need_update}, updated: {short_updated})") + print(f"Total files needing update: {need_update}") + print(f"Total files updated: {updated}") + + if need_update > 0 and updated == 0: + sys.exit(1) + sys.exit(0) +if __name__ == "__main__": + main() diff --git a/tests/instructions/test_print_tensor.py b/tests/instructions/test_print_tensor.py index f156f796..9dc81c81 100644 --- a/tests/instructions/test_print_tensor.py +++ b/tests/instructions/test_print_tensor.py @@ -1,3 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import tilus from tilus import float32 from tilus.testing import requires From 5835696367e97607301002c79a88b501b8190b0b Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 16:11:01 -0400 Subject: [PATCH 2/9] wip Signed-off-by: Yaoyao Ding --- .github/actions/setup-environment/action.yaml | 4 +- .github/workflows/deploy-docs.yaml | 2 +- .github/workflows/format-and-lint.yaml | 2 +- .github/workflows/scripts/run-examples.sh | 5 +- .github/workflows/tests.yaml | 10 +- README.md | 8 +- docs/source/_static/custom.css | 2 +- .../getting-started/tutorials/.gitignore | 2 +- docs/source/programming-guides/autotuning.rst | 1 - docs/source/programming-guides/cache.rst | 1 - .../programming-guides/control-flow.rst | 1 - .../layout-system/__init__.rst | 1 - .../layout-system/layout-inference.rst | 2 +- .../programming-guides/thread-group.rst | 38 ++--- .../type-system/pointer-types.rst | 2 - .../type-system/scalar-types.rst | 4 - docs/source/python-api/tilus-ir/__index__.rst | 1 - .../python-api/tilus-ir/global_layout.rst | 1 - .../python-api/tilus-ir/global_tensor.rst | 1 - .../python-api/tilus-ir/register_tensor.rst | 1 - .../python-api/tilus-ir/shared_layout.rst | 1 - .../python-api/tilus-ir/shared_tensor.rst | 1 - docs/source/python-api/tilus-ir/tensor.rst | 2 - .../include/hidet/tvm/ffi/extra_type_traits.h | 16 +- .../extensions/hidet/include/hidet/void_p.h | 2 +- .../hidet/ir/primitives/cuda/tensor_map.py | 22 +-- .../extensions/hidet/ir/primitives/runtime.py | 1 - .../tilus/extensions/hidet/utils/ncu_utils.py | 26 ++-- python/tilus/lang/modules/cuda.py | 2 +- python/tilus/lang/transpiler.py | 4 +- python/tilus/target.py | 8 +- .../transforms/instruments/utils/highlight.py | 6 +- python/tilus/utils/cuda_blocking_run.py | 2 +- scripts/add-copyright.py | 147 ------------------ scripts/{ => lint}/format-and-lint.sh | 0 scripts/{ => lint}/lock-gpu-clocks.py | 0 36 files changed, 80 insertions(+), 249 deletions(-) delete mode 100644 scripts/add-copyright.py rename scripts/{ => lint}/format-and-lint.sh (100%) rename scripts/{ => lint}/lock-gpu-clocks.py (100%) diff --git a/.github/actions/setup-environment/action.yaml b/.github/actions/setup-environment/action.yaml index fb2879a8..51211b0b 100644 --- a/.github/actions/setup-environment/action.yaml +++ b/.github/actions/setup-environment/action.yaml @@ -32,7 +32,7 @@ runs: } # Install required packages - install_package git + install_package git install_package cmake shell: bash @@ -45,7 +45,7 @@ runs: uses: actions/setup-python@v4 with: python-version: ${{ inputs.python-version }} - + - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main if: hashFiles('/etc/buildkit/buildkitd.toml') != '' diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml index dea26173..7d392f0e 100644 --- a/.github/workflows/deploy-docs.yaml +++ b/.github/workflows/deploy-docs.yaml @@ -10,7 +10,7 @@ concurrency: permissions: contents: read pages: write - id-token: write + id-token: write jobs: build: diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index f9622b83..1009f3f8 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -2,7 +2,7 @@ name: Format & Lint on: push: - branches: + branches: - "pull-request/[0-9]+" concurrency: diff --git a/.github/workflows/scripts/run-examples.sh b/.github/workflows/scripts/run-examples.sh index 640239fa..e56bdcaa 100644 --- a/.github/workflows/scripts/run-examples.sh +++ b/.github/workflows/scripts/run-examples.sh @@ -38,7 +38,7 @@ echo "Found $(echo "$pyfiles" | wc -l) Python files" # Loop through each file more traditionally for script in $pyfiles; do ((total_scripts++)) - + if ! run_script "$script"; then failed_scripts+=("$script") ((failed_count++)) @@ -64,6 +64,3 @@ else echo "All scripts passed successfully!" exit 0 fi - - - diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 781c1a54..cab6a437 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,7 +2,7 @@ name: PR Tests on: push: - branches: + branches: - "pull-request/[0-9]+" concurrency: @@ -42,7 +42,7 @@ jobs: examples/** pyproject.toml base_sha: 'origin/main' - + - name: Check for changes in docs id: changed-docs uses: step-security/changed-files@v46 @@ -59,7 +59,7 @@ jobs: continue-on-error: true strategy: matrix: - runner: + runner: - linux-amd64-gpu-l4-latest-1 runs-on: ${{ matrix.runner }} container: @@ -127,7 +127,7 @@ jobs: run: | pip install wheel pip install torch==2.8 # since flash-attn only has pre-built wheels for torch<=2.8, todo: update this when flash-attn supports torch>=2.9 - pip install flash-attn --no-build-isolation + pip install flash-attn --no-build-isolation - name: Run examples run: | @@ -144,7 +144,7 @@ jobs: echo "Tests result: ${{ needs.tests.result }}" echo "Docs result: ${{ needs.docs.result }}" echo "Examples result: ${{ needs.examples.result }}" - + # Check if any required job failed if [[ "${{ needs.tests.result }}" == "failure" ]] || [[ "${{ needs.docs.result }}" == "failure" ]] || [[ "${{ needs.examples.result }}" == "failure" ]]; then echo "One or more functional tests failed" diff --git a/README.md b/README.md index bed7a3cf..2a94b71c 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,10 @@ Tilus is pronounced as tie-lus, /ˈtaɪləs/. ### Installation Install Tilus using `pip`: ``` -pip install tilus +pip install tilus ``` -> [!NOTE] +> [!NOTE] > Tilus depends on `cuda-python`. If your GPU driver is older than **580.65.06**, you will need to install an older version of cuda-python to ensure compatibility. > ``` > pip install tilus "cuda-python<13" @@ -27,7 +27,7 @@ pip install tilus ### Usage -To get started, refer to the [tutorials](https://nvidia.github.io/tilus/getting-started/tutorials/__init__.html) to learn how to program kernels with Tilus. +To get started, refer to the [tutorials](https://nvidia.github.io/tilus/getting-started/tutorials/__init__.html) to learn how to program kernels with Tilus. You can also check more [examples](https://github.com/NVIDIA/tilus/tree/main/examples) of using Tilus. @@ -50,5 +50,5 @@ This project is based on the following research paper: We would like to acknowledge the following projects for their influence on Tilus's design and development: - **Hidet**: We take Hidet IR as our low-level target and reuse its runtime system. - **TVM**: Hidet's initial IR was adopted from TVM, and we also learned a lot from TVM on how to build a compiler. -- **Triton**: The core idea of defining kernels at a thread-block level and working with tiles was inspired by Triton. +- **Triton**: The core idea of defining kernels at a thread-block level and working with tiles was inspired by Triton. - **Hexcute**: We adopted the idea of using automatic layout inference to simplify programming from Hexcute. diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 6277c6d4..088d4992 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -2,4 +2,4 @@ See: https://github.com/executablebooks/sphinx-book-theme/issues/732 */ #rtd-footer-container { margin: 0px !important; -} \ No newline at end of file +} diff --git a/docs/source/getting-started/tutorials/.gitignore b/docs/source/getting-started/tutorials/.gitignore index 2846015a..65bbfc38 100644 --- a/docs/source/getting-started/tutorials/.gitignore +++ b/docs/source/getting-started/tutorials/.gitignore @@ -1 +1 @@ -matmul/ \ No newline at end of file +matmul/ diff --git a/docs/source/programming-guides/autotuning.rst b/docs/source/programming-guides/autotuning.rst index 3f28b425..d8edc695 100644 --- a/docs/source/programming-guides/autotuning.rst +++ b/docs/source/programming-guides/autotuning.rst @@ -68,4 +68,3 @@ When we launch the kernel, tilus will automatically compile the kernel with all The kernels will be compiled in parallel when we first call the kernel with a specific input size triggering the JIT compilation (:doc:`tilus-script`). We can use :py:func:`tilus.option.parallel_workers` to control the number of parallel workers to compile the kernels. - diff --git a/docs/source/programming-guides/cache.rst b/docs/source/programming-guides/cache.rst index e6256801..3795f513 100644 --- a/docs/source/programming-guides/cache.rst +++ b/docs/source/programming-guides/cache.rst @@ -70,4 +70,3 @@ The cache directory contains the following structure (only show the important pa - **options.txt** the options used to compile the program - ... - diff --git a/docs/source/programming-guides/control-flow.rst b/docs/source/programming-guides/control-flow.rst index 0d9c1eb2..04a5e269 100644 --- a/docs/source/programming-guides/control-flow.rst +++ b/docs/source/programming-guides/control-flow.rst @@ -95,4 +95,3 @@ In addition to the control flow statements mentioned above, Tilus Script also su When encountered, it will immediately terminate the innermost loop and continue execution after the loop. - **continue**: This statement can be used to skip the current iteration of a loop and continue with the next iteration. When encountered, it will immediately jump to the next iteration of the innermost loop. - diff --git a/docs/source/programming-guides/layout-system/__init__.rst b/docs/source/programming-guides/layout-system/__init__.rst index 7ea07808..9b74e424 100644 --- a/docs/source/programming-guides/layout-system/__init__.rst +++ b/docs/source/programming-guides/layout-system/__init__.rst @@ -29,4 +29,3 @@ We require the user to explicitly or implicitly define the layout of global tens the kernel and the host. As for shared and register tensors, the layout can be explicitly defined by the user, or automatically inferred by the tilus compiler based on the usage of the tensor. We give a brief overview of the layout inference algorithm adopted by tilus in :doc:`layout-inference`. - diff --git a/docs/source/programming-guides/layout-system/layout-inference.rst b/docs/source/programming-guides/layout-system/layout-inference.rst index 067b8d63..3fa61bb0 100644 --- a/docs/source/programming-guides/layout-system/layout-inference.rst +++ b/docs/source/programming-guides/layout-system/layout-inference.rst @@ -34,4 +34,4 @@ validation phase. Each instruction has a pre-defined validation rule that checks If we inferred all layouts successfully and all of them are valid, we will proceed to subsequent steps of compilation. If you are interested in the details, feel free to check the source code of the layout inference algorithm in -:py:mod:`tilus.ir.layout.inference`. \ No newline at end of file +:py:mod:`tilus.ir.layout.inference`. diff --git a/docs/source/programming-guides/thread-group.rst b/docs/source/programming-guides/thread-group.rst index c1648872..4ce93fdf 100644 --- a/docs/source/programming-guides/thread-group.rst +++ b/docs/source/programming-guides/thread-group.rst @@ -61,24 +61,24 @@ Basic Usage Examples class MyScript(tilus.Script): def __init__(self): super().__init__() - + def __call__(self, ...): # Specify 4 warps = 128 threads total self.attrs.warps = 4 - + # All threads execute this data = self.register_tensor(dtype=f32, shape=[16]) - + # Only threads in group 0 execute this block (threads 0-31) with self.thread_group(0, num_groups=4): # Instructions for first quarter of threads result = self.load_global(src, offsets=[0, 0]) - + # Only threads in group 1 execute this block (threads 32-63) with self.thread_group(1, num_groups=4): # Instructions for second quarter of threads result = self.load_global(src, offsets=[16, 0]) - + # All threads execute this again self.sync() @@ -89,17 +89,17 @@ Basic Usage Examples class MyScript(tilus.Script): def __init__(self): super().__init__() - + def __call__(self, ...): # Specify 4 warps = 128 threads total self.attrs.warps = 4 - + # Only threads in group 0 execute this (32 threads: 0-31) with self.thread_group(0, group_size=32): # First group of 32 threads processes first chunk data = self.load_global(src, offsets=[0, 0]) - - # Only threads in group 1 execute this (32 threads: 32-63) + + # Only threads in group 1 execute this (32 threads: 32-63) with self.thread_group(1, group_size=32): # Second group of 32 threads processes second chunk data = self.load_global(src, offsets=[32, 0]) @@ -111,24 +111,24 @@ Basic Usage Examples class MyScript(tilus.Script): def __init__(self): super().__init__() - + def __call__(self, ...): # Specify 4 warps = 128 threads total self.attrs.warps = 4 - + # First level: split into 4 groups of 32 threads each with self.thread_group(0, num_groups=4): # Only first group (threads 0-31) enters here - + # Second level: further split into 2 sub-groups of 16 threads each with self.thread_group(0, num_groups=2): # Only threads 0-15 execute this fine_grained_work() - + with self.thread_group(1, num_groups=2): # Only threads 16-31 execute this different_fine_grained_work() - + # Back to first level - all threads 0-31 execute this self.sync() @@ -142,20 +142,20 @@ The ``self.sync()`` instruction synchronizes all threads in the **current thread class MyScript(tilus.Script): def __init__(self): super().__init__() - + def __call__(self, ...): # Specify 4 warps = 128 threads total self.attrs.warps = 4 - + with self.thread_group(0, num_groups=2): # Some work by first half of threads (threads 0-63) work_part_1() - + # Synchronize only threads in group 0 self.sync() # Only waits for threads 0-63 - + # Continue with synchronized work work_part_2() - + # Synchronize all threads in the thread block self.sync() diff --git a/docs/source/programming-guides/type-system/pointer-types.rst b/docs/source/programming-guides/type-system/pointer-types.rst index fb513288..e55e9808 100644 --- a/docs/source/programming-guides/type-system/pointer-types.rst +++ b/docs/source/programming-guides/type-system/pointer-types.rst @@ -20,5 +20,3 @@ Where ``dtype`` is one of the scalar types (e.g., :py:data:`~tilus.float32`) in **Void Pointer** A special pointer type :py:data:`tilus.ir.void_p`, which represents a pointer to an unspecified data type, is often used as a generic pointer type. - - diff --git a/docs/source/programming-guides/type-system/scalar-types.rst b/docs/source/programming-guides/type-system/scalar-types.rst index 9fd014e7..7af93904 100644 --- a/docs/source/programming-guides/type-system/scalar-types.rst +++ b/docs/source/programming-guides/type-system/scalar-types.rst @@ -64,7 +64,3 @@ Tilus supports the following data types. All of them are instances of the :class - :py:data:`tilus.float5_e2m2` - :py:data:`tilus.float4_e2m1` - :py:data:`tilus.float3_e1m1` - - - - diff --git a/docs/source/python-api/tilus-ir/__index__.rst b/docs/source/python-api/tilus-ir/__index__.rst index 19519692..82c4ecf0 100644 --- a/docs/source/python-api/tilus-ir/__index__.rst +++ b/docs/source/python-api/tilus-ir/__index__.rst @@ -28,4 +28,3 @@ tilus.ir register_layout shared_layout global_layout - diff --git a/docs/source/python-api/tilus-ir/global_layout.rst b/docs/source/python-api/tilus-ir/global_layout.rst index adaf4334..e312be78 100644 --- a/docs/source/python-api/tilus-ir/global_layout.rst +++ b/docs/source/python-api/tilus-ir/global_layout.rst @@ -21,4 +21,3 @@ We can use the following functions to create a global layout: global_column_major global_strides global_compose - diff --git a/docs/source/python-api/tilus-ir/global_tensor.rst b/docs/source/python-api/tilus-ir/global_tensor.rst index f57d647e..b1bfd476 100644 --- a/docs/source/python-api/tilus-ir/global_tensor.rst +++ b/docs/source/python-api/tilus-ir/global_tensor.rst @@ -5,4 +5,3 @@ tilus.ir.GlobalTensor .. autoclass:: tilus.ir.GlobalTensor :members: shape, layout, size, __getitem__ :exclude-members: __init__, __new__ - diff --git a/docs/source/python-api/tilus-ir/register_tensor.rst b/docs/source/python-api/tilus-ir/register_tensor.rst index abdf1951..18d4d3c2 100644 --- a/docs/source/python-api/tilus-ir/register_tensor.rst +++ b/docs/source/python-api/tilus-ir/register_tensor.rst @@ -5,4 +5,3 @@ tilus.ir.RegisterTensor .. autoclass:: tilus.ir.RegisterTensor :members: shape, optional_layout, __add__, __sub__, __mul__, __truediv__, create, layout, squeeze, unsqueeze, transpose, to :exclude-members: __init__, __new__ - diff --git a/docs/source/python-api/tilus-ir/shared_layout.rst b/docs/source/python-api/tilus-ir/shared_layout.rst index 05f9d4f6..19856c33 100644 --- a/docs/source/python-api/tilus-ir/shared_layout.rst +++ b/docs/source/python-api/tilus-ir/shared_layout.rst @@ -19,4 +19,3 @@ We can use the following functions to create a shared layout: shared_row_major shared_column_major shared_compose - diff --git a/docs/source/python-api/tilus-ir/shared_tensor.rst b/docs/source/python-api/tilus-ir/shared_tensor.rst index 21434cf6..78711429 100644 --- a/docs/source/python-api/tilus-ir/shared_tensor.rst +++ b/docs/source/python-api/tilus-ir/shared_tensor.rst @@ -4,4 +4,3 @@ tilus.ir.SharedTensor .. autoclass:: tilus.ir.SharedTensor :members: dtype, shape, optional_layout, layout, size, nbytes :exclude-members: __init__, __new__ - diff --git a/docs/source/python-api/tilus-ir/tensor.rst b/docs/source/python-api/tilus-ir/tensor.rst index bc285b5a..51a094dc 100644 --- a/docs/source/python-api/tilus-ir/tensor.rst +++ b/docs/source/python-api/tilus-ir/tensor.rst @@ -4,5 +4,3 @@ tilus.ir.Tensor .. autoclass:: tilus.ir.Tensor :members: dtype, as_register_tensor, as_shared_tensor, as_global_tensor, as_register_or_shared_tensor :exclude-members: __init__, __new__ - - diff --git a/python/tilus/extensions/hidet/include/hidet/tvm/ffi/extra_type_traits.h b/python/tilus/extensions/hidet/include/hidet/tvm/ffi/extra_type_traits.h index 5c2b715c..6708040f 100644 --- a/python/tilus/extensions/hidet/include/hidet/tvm/ffi/extra_type_traits.h +++ b/python/tilus/extensions/hidet/include/hidet/tvm/ffi/extra_type_traits.h @@ -26,7 +26,7 @@ inline std::string dtype_to_str(DLDataType dtype) { template <> struct TypeTraits : public FallbackOnlyTraitsBase { - TVM_FFI_INLINE static std::string TypeStr() { return "void_p"; } + TVM_FFI_INLINE static std::string TypeStr() { return "void_p"; } TVM_FFI_INLINE static void_p ConvertFallbackValue(DLTensor* src) { return src->data; @@ -41,10 +41,10 @@ struct TypeTraits : public FallbackOnlyTraitsBase struct TypeTraits : public FallbackOnlyTraitsBase { - TVM_FFI_INLINE static std::string TypeStr() { return "float16*"; } + TVM_FFI_INLINE static std::string TypeStr() { return "float16*"; } TVM_FFI_INLINE static half* ConvertFallbackValue(DLTensor* src) { if (src->dtype.code != kDLFloat || src->dtype.bits != 16) { @@ -57,7 +57,7 @@ struct TypeTraits : public FallbackOnlyTraitsBase { // Template specialization for __nv_bfloat16* template <> struct TypeTraits<__nv_bfloat16*> : public FallbackOnlyTraitsBase<__nv_bfloat16*, DLTensor*> { - TVM_FFI_INLINE static std::string TypeStr() { return "bfloat16*"; } + TVM_FFI_INLINE static std::string TypeStr() { return "bfloat16*"; } TVM_FFI_INLINE static __nv_bfloat16* ConvertFallbackValue(DLTensor* src) { if (src->dtype.code != kDLBfloat || src->dtype.bits != 16) { @@ -70,7 +70,7 @@ struct TypeTraits<__nv_bfloat16*> : public FallbackOnlyTraitsBase<__nv_bfloat16* // Template specialization for float*, double* template struct TypeTraits>> : public FallbackOnlyTraitsBase { - TVM_FFI_INLINE static std::string TypeStr() { return "float" + std::to_string(sizeof(Float) * 8); } + TVM_FFI_INLINE static std::string TypeStr() { return "float" + std::to_string(sizeof(Float) * 8); } TVM_FFI_INLINE static Float* ConvertFallbackValue(DLTensor* src) { if (src->dtype.code != kDLFloat || src->dtype.bits != sizeof(Float) * 8) { @@ -83,7 +83,7 @@ struct TypeTraits>> : p // Template specialization for int32_t*, int16_t*, etc. template struct TypeTraits && std::is_integral_v>> : public FallbackOnlyTraitsBase { - TVM_FFI_INLINE static std::string TypeStr() { return "int" + std::to_string(sizeof(Int) * 8); } + TVM_FFI_INLINE static std::string TypeStr() { return "int" + std::to_string(sizeof(Int) * 8); } TVM_FFI_INLINE static Int* ConvertFallbackValue(DLTensor* src) { if (src->dtype.code != kDLInt || src->dtype.bits != sizeof(Int) * 8) { @@ -96,10 +96,10 @@ struct TypeTraits && std::is_integr // Template specialization for uint32_t*, uint16_t*, etc. template struct TypeTraits && std::is_integral_v>> : public FallbackOnlyTraitsBase { - TVM_FFI_INLINE static std::string TypeStr() { return "uint" + std::to_string(sizeof(UInt) * 8); } + TVM_FFI_INLINE static std::string TypeStr() { return "uint" + std::to_string(sizeof(UInt) * 8); } TVM_FFI_INLINE static UInt* ConvertFallbackValue(DLTensor* src) { - if ((src->dtype.code != kDLUInt || src->dtype.bits != sizeof(UInt) * 8) + if ((src->dtype.code != kDLUInt || src->dtype.bits != sizeof(UInt) * 8) && (src->dtype.code != kDLBool || src->dtype.bits != 8) ) { TVM_FFI_THROW(ValueError) << "Expect a tensor with " << sizeof(UInt) * 8 << " bit unsigned integer, got a tensor with dtype " << dtype_to_str(src->dtype); diff --git a/python/tilus/extensions/hidet/include/hidet/void_p.h b/python/tilus/extensions/hidet/include/hidet/void_p.h index 0b793cb1..1b9ccb12 100644 --- a/python/tilus/extensions/hidet/include/hidet/void_p.h +++ b/python/tilus/extensions/hidet/include/hidet/void_p.h @@ -12,7 +12,7 @@ class void_p { template void_p(T *ptr) : internal_ptr(static_cast(ptr)) {} - // 3. Implicit template conversion operator (Allows implicit conversion back to ANY T*) This allows: int* ip = p; + // 3. Implicit template conversion operator (Allows implicit conversion back to ANY T*) This allows: int* ip = p; // (where p holds an int*) template operator T *() const { return static_cast(internal_ptr); diff --git a/python/tilus/extensions/hidet/ir/primitives/cuda/tensor_map.py b/python/tilus/extensions/hidet/ir/primitives/cuda/tensor_map.py index f779dd8d..52c9003e 100644 --- a/python/tilus/extensions/hidet/ir/primitives/cuda/tensor_map.py +++ b/python/tilus/extensions/hidet/ir/primitives/cuda/tensor_map.py @@ -145,17 +145,17 @@ def encode_tensor_map( """ template_string = ( """cuTensorMapEncodeTiled( - {{}}, - {dtype}, - {{}}, - {{}}, - {{}}, - {{}}, - {{}}, - {{}}, - {interleave}, - {swizzle}, - {l2_promotion}, + {{}}, + {dtype}, + {{}}, + {{}}, + {{}}, + {{}}, + {{}}, + {{}}, + {interleave}, + {swizzle}, + {l2_promotion}, {oob_fill} ); """.format( diff --git a/python/tilus/extensions/hidet/ir/primitives/runtime.py b/python/tilus/extensions/hidet/ir/primitives/runtime.py index bdf32338..2143c002 100644 --- a/python/tilus/extensions/hidet/ir/primitives/runtime.py +++ b/python/tilus/extensions/hidet/ir/primitives/runtime.py @@ -23,4 +23,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/tilus/extensions/hidet/utils/ncu_utils.py b/python/tilus/extensions/hidet/utils/ncu_utils.py index 3f2ff21f..c0fdff29 100644 --- a/python/tilus/extensions/hidet/utils/ncu_utils.py +++ b/python/tilus/extensions/hidet/utils/ncu_utils.py @@ -31,20 +31,20 @@ --kernel-name regex:"{kernel_regex}" --force-overwrite --set full ---rule CPIStall ---rule FPInstructions ---rule HighPipeUtilization ---rule IssueSlotUtilization ---rule LaunchConfiguration ---rule Occupancy ---rule PCSamplingData ---rule SOLBottleneck ---rule SOLFPRoofline ---rule SharedMemoryConflicts ---rule SlowPipeLimiter ---rule ThreadDivergence +--rule CPIStall +--rule FPInstructions +--rule HighPipeUtilization +--rule IssueSlotUtilization +--rule LaunchConfiguration +--rule Occupancy +--rule PCSamplingData +--rule SOLBottleneck +--rule SOLFPRoofline +--rule SharedMemoryConflicts +--rule SlowPipeLimiter +--rule ThreadDivergence --rule UncoalescedGlobalAccess ---rule UncoalescedSharedAccess +--rule UncoalescedSharedAccess --import-source yes --check-exit-code yes {python_executable} {python_script} {args} diff --git a/python/tilus/lang/modules/cuda.py b/python/tilus/lang/modules/cuda.py index 3b6c9175..2d2e13bf 100644 --- a/python/tilus/lang/modules/cuda.py +++ b/python/tilus/lang/modules/cuda.py @@ -336,7 +336,7 @@ def _swizzled_shared_layout(dtype: DataType, shape: tuple[int, ...]) -> SharedLa core = shared_row_major(rows, 2).swizzle(dim=1, regards_dim=0, log_step=2) else: """ - 0 + 0 1 2 3 diff --git a/python/tilus/lang/transpiler.py b/python/tilus/lang/transpiler.py index 3e608eea..9d783186 100644 --- a/python/tilus/lang/transpiler.py +++ b/python/tilus/lang/transpiler.py @@ -543,10 +543,10 @@ def visit_Call(self, expr: ast.Call) -> Any: """ There are different kinds of function calls in Tilus Script: 1. inlined kernel procedure, it is a method of the user-defined Script subclass - 2. (global, shared or register) tensor method, such as `tensor.to(dtype)`, etc. + 2. (global, shared or register) tensor method, such as `tensor.to(dtype)`, etc. 3. python builtin function, such as `max`, `min`, for scalar expressions. 4. other function/method calls - + We treat 1 to 3 specially, and call the function directly in 4. """ diff --git a/python/tilus/target.py b/python/tilus/target.py index ee398500..e2f7a254 100644 --- a/python/tilus/target.py +++ b/python/tilus/target.py @@ -82,13 +82,13 @@ def supports(self, target): """ Predefined targets - + The generic ones: - gpgpu/any: any GPU - amdgpu/any: any AMD GPU - nvgpu/any: any NVIDIA GPU are used to represent the generic targets that our compilation process (like scheduler) can work on. - + Each specific GPU must be represented by a specific target, e.g., amdgpu/gfx1100 for AMD RX 7900 XTX. """ gpgpu_any = Target(kind="gpgpu", arch="any", properties=TargetProperties()) @@ -111,9 +111,9 @@ def supports(self, target): """ NVIDIA GPUs - + See Also: https://docs.nvidia.com/cuda/cuda-c-programming-guide/#compute-capabilities - + Suffixes: - No suffix: Base architecture - 'f': Family variant diff --git a/python/tilus/transforms/instruments/utils/highlight.py b/python/tilus/transforms/instruments/utils/highlight.py index 8193f343..59f1cce5 100644 --- a/python/tilus/transforms/instruments/utils/highlight.py +++ b/python/tilus/transforms/instruments/utils/highlight.py @@ -163,11 +163,11 @@ def split_into_lines(tokens: List[Tuple[TokenKind, str]]) -> List[List[Tuple[Tok background-color: #e0e0e0; color: #000; text-decoration: none; - } + } main { padding: 2em; - margin-left: 200px; - padding-left: 2em; + margin-left: 200px; + padding-left: 2em; flex-grow: 1; overflow-x: auto; } diff --git a/python/tilus/utils/cuda_blocking_run.py b/python/tilus/utils/cuda_blocking_run.py index 8dc63639..c281f7d8 100644 --- a/python/tilus/utils/cuda_blocking_run.py +++ b/python/tilus/utils/cuda_blocking_run.py @@ -21,7 +21,7 @@ import sys _cuda_blocking_template = """ - CUDA_LAUNCH_BLOCKING=1 {python_executable} {python_script} {args} + CUDA_LAUNCH_BLOCKING=1 {python_executable} {python_script} {args} """.replace("\n", " ").strip() diff --git a/scripts/add-copyright.py b/scripts/add-copyright.py deleted file mode 100644 index d809b6d9..00000000 --- a/scripts/add-copyright.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from datetime import datetime - -current_year = datetime.now().year - -LICENSE_HEADER = """ -# SPDX-FileCopyrightText: Copyright (c) {year} NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""".format(year=current_year) - -SHORT_LICENSE_HEADER = """ -# SPDX-FileCopyrightText: Copyright (c) {year} NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -""".format(year=current_year) - -LICENSE_LINE = "# SPDX-License-Identifier: Apache-2.0\n" -TARGET_DIRS = [ - os.path.join(os.path.dirname(__file__), "..", "python"), - os.path.join(os.path.dirname(__file__), "..", "tests"), - os.path.join(os.path.dirname(__file__), "..", "scripts"), -] -SHORT_LICENSE_TARGET_DIRS = [ - os.path.join(os.path.dirname(__file__), "..", "examples"), -] - - -def add_license_to_file(filepath, filetype, use_short_header=False): - with open(filepath, "r", encoding="utf-8") as f: - lines = f.readlines() - - # Choose which header to use - header_to_use = SHORT_LICENSE_HEADER if use_short_header else LICENSE_HEADER - - # Find where to insert (after shebang if present, else at top) - insert_idx = 1 if lines and lines[0].startswith("#!") else 0 - - # Check first 15 lines for any existing license header - search_lines = lines[:15] - - # Check if any header already exists (with SPDX-FileCopyrightText) - has_any_header = any("SPDX-FileCopyrightText" in line for line in search_lines) - if has_any_header: - return False # Header already exists, don't modify - - # Check if only the short license line exists - has_license_line = any(LICENSE_LINE.strip() in line for line in search_lines) - - if has_license_line: - # Replace the short license line with the appropriate header - # Find and remove the existing short license line - for i, line in enumerate(lines): - if LICENSE_LINE.strip() in line: - lines.pop(i) - break - # Insert the appropriate header at the right position - header_lines = header_to_use.split("\n") - for i, header_line in enumerate(reversed(header_lines)): - if header_line: # Skip empty lines at the end - lines.insert(insert_idx, header_line + "\n") - else: - # No license header exists, add the appropriate header - header_lines = header_to_use.split("\n") - for i, header_line in enumerate(reversed(header_lines)): - if header_line: # Skip empty lines at the end - lines.insert(insert_idx, header_line + "\n") - - with open(filepath, "w", encoding="utf-8") as f: - f.writelines(lines) - return True - - -def process_directory(target_dirs, use_short_header=False): - total_files = 0 - need_update = 0 - updated = 0 - - for target_dir in target_dirs: - for root, _, files in os.walk(target_dir): - for file in files: - if file.endswith(".py") or file.endswith(".sh"): - total_files += 1 - filepath = os.path.join(root, file) - with open(filepath, "r", encoding="utf-8") as f: - lines = f.readlines() - - # Check first 15 lines for any existing license header - search_lines = lines[:15] - has_any_header = any("SPDX-FileCopyrightText" in line for line in search_lines) - - # File needs update if it has no header - if not has_any_header: - need_update += 1 - if add_license_to_file( - filepath, filetype=file.split(".")[-1], use_short_header=use_short_header - ): - updated += 1 - - return total_files, need_update, updated - - -def main(): - # Process directories with full license header - full_total, full_need_update, full_updated = process_directory(TARGET_DIRS, use_short_header=False) - - # Process directories with short license header - short_total, short_need_update, short_updated = process_directory(SHORT_LICENSE_TARGET_DIRS, use_short_header=True) - - # Combined totals - total_files = full_total + short_total - need_update = full_need_update + short_need_update - updated = full_updated + short_updated - - print(f"Total source files found (.py/.sh): {total_files}") - print(f"Files with full header: {full_total} (need update: {full_need_update}, updated: {full_updated})") - print(f"Files with short header: {short_total} (need update: {short_need_update}, updated: {short_updated})") - print(f"Total files needing update: {need_update}") - print(f"Total files updated: {updated}") - - -if __name__ == "__main__": - main() diff --git a/scripts/format-and-lint.sh b/scripts/lint/format-and-lint.sh similarity index 100% rename from scripts/format-and-lint.sh rename to scripts/lint/format-and-lint.sh diff --git a/scripts/lock-gpu-clocks.py b/scripts/lint/lock-gpu-clocks.py similarity index 100% rename from scripts/lock-gpu-clocks.py rename to scripts/lint/lock-gpu-clocks.py From 10dc5201cad8936f46c5fb0bbe68df02b8921573 Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 16:27:41 -0400 Subject: [PATCH 3/9] wip Signed-off-by: Yaoyao Ding --- .pre-commit-config.yaml | 16 +++++----- scripts/lint/check-copyright-header.py | 44 ++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 727fb2fc..23e0bf34 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,18 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-added-large-files - repo: local hooks: - id: check-copyright-header name: Check copyright headers - entry: python scripts/lint/check-copyright-header.py --check + entry: python scripts/lint/check-copyright-header.py --check-and-fix language: system types: [python] pass_filenames: false +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files diff --git a/scripts/lint/check-copyright-header.py b/scripts/lint/check-copyright-header.py index cbb54166..85dbf79e 100644 --- a/scripts/lint/check-copyright-header.py +++ b/scripts/lint/check-copyright-header.py @@ -135,20 +135,30 @@ def main(): help="Only check if files have headers, don't modify them") parser.add_argument("--fix", action="store_true", help="Automatically add missing headers to files") + parser.add_argument("--check-and-fix", action="store_true", + help="Check and fix missing headers in files") args = parser.parse_args() - # Default behavior is fix if neither check nor fix is specified - if not args.check and not args.fix: + # Ensure only one of the three arguments is set + selected_args = [args.check, args.fix, args.check_and_fix] + if sum(selected_args) > 1: + parser.error("Only one of --check, --fix, or --check-and-fix can be specified") + + # Default behavior is fix if no arguments are specified + if not any(selected_args): args.fix = True - check_only = args.check # Process directories with full license header + check_only = args.check + check_and_fix = args.check_and_fix + + # Process directories with full license header full_total, full_need_update, full_updated, full_files_needing_update = process_directory( - TARGET_DIRS, use_short_header=False, check_only=check_only) + TARGET_DIRS, use_short_header=False, check_only=check_only or check_and_fix) # Process directories with short license header short_total, short_need_update, short_updated, short_files_needing_update = process_directory( - SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=check_only) + SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=check_only or check_and_fix) # Combined totals total_files = full_total + short_total @@ -161,11 +171,33 @@ def main(): print(f"Found {need_update} files missing copyright headers:") for filepath in all_files_needing_update: print(f" {filepath}") + print("Running the following command to fix the issues:") + print(" python scripts/lint/check-copyright-header.py --fix") sys.exit(1) else: print(f"All {total_files} source files have copyright headers.") sys.exit(0) - else: + elif check_and_fix: + if need_update > 0: + # Files need fixing, so run the fix process + print(f"Found {need_update} files missing copyright headers. Fixing them...") + + # Process directories again with fix mode + full_total_fix, full_need_update_fix, full_updated_fix, _ = process_directory( + TARGET_DIRS, use_short_header=False, check_only=False) + short_total_fix, short_need_update_fix, short_updated_fix, _ = process_directory( + SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=False) + + total_updated_fix = full_updated_fix + short_updated_fix + + print(f"Fixed copyright headers in {total_updated_fix} files:") + for filepath in all_files_needing_update: + print(f" {filepath}") + sys.exit(1) + else: + print(f"All {total_files} source files have copyright headers.") + sys.exit(0) + else: # --fix mode print(f"Total source files found (.py/.sh): {total_files}") print(f"Files with full header: {full_total} (need update: {full_need_update}, updated: {full_updated})") print(f"Files with short header: {short_total} (need update: {short_need_update}, updated: {short_updated})") From c5803d752382b31bdbb10276345a13da7c526f3b Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 16:44:37 -0400 Subject: [PATCH 4/9] wip Signed-off-by: Yaoyao Ding --- .pre-commit-config.yaml | 29 ++++++++++- docs/source/conf.py | 51 +++++++++---------- python/tilus/ir/layout/mfunction/mfunction.py | 4 +- python/tilus/ir/layout/register_layout.py | 4 +- python/tilus/ir/layout/utils/cute.py | 2 +- python/tilus/utils/__init__.py | 4 +- scripts/lint/check-copyright-header.py | 23 +++++---- 7 files changed, 71 insertions(+), 46 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 23e0bf34..cd74a4ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,33 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version + rev: v0.12.3 + hooks: + # Run the linter with automatic fixes - uses [tool.ruff] config from pyproject.toml + - id: ruff-check + name: Ruff linter (with fixes) + args: [--fix] + types_or: [python, pyi, jupyter] + # Run the formatter - uses [tool.ruff] config from pyproject.toml + - id: ruff-format + name: Ruff formatter + types_or: [python, pyi, jupyter] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.15.0 + hooks: + - id: mypy + name: MyPy type checking + # Uses [tool.mypy] configuration from pyproject.toml + additional_dependencies: [ + types-tabulate, + types-tqdm, + ] + # Only check files in the python package (matches original script scope) + files: ^python/ + # Exclude hidet extensions (they have special mypy overrides in pyproject.toml) + exclude: ^python/tilus/extensions/hidet/ - repo: local hooks: - id: check-copyright-header @@ -10,7 +37,7 @@ repos: types: [python] pass_filenames: false - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/docs/source/conf.py b/docs/source/conf.py index 133d6230..248b0707 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -2,21 +2,16 @@ # # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -import os -import glob -import shutil -import sphinx_gallery.sorting -import tilus.utils # from sphinx_gallery.sorting import FileNameSortKey # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'tilus' -copyright = '2025, NVIDIA' -author = 'NVIDIA' +project = "tilus" +copyright = "2025, NVIDIA" +author = "NVIDIA" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration @@ -25,30 +20,30 @@ "sphinx.ext.napoleon", "sphinx.ext.autodoc", "sphinx.ext.autosummary", - 'sphinx.ext.intersphinx', - 'sphinx.ext.viewcode', - 'sphinx.ext.coverage', - 'sphinx.ext.todo', - 'sphinx.ext.graphviz', - 'sphinx.ext.doctest', - 'sphinx_copybutton', - 'autodocsumm', - 'sphinx_gallery.gen_gallery' + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx.ext.coverage", + "sphinx.ext.todo", + "sphinx.ext.graphviz", + "sphinx.ext.doctest", + "sphinx_copybutton", + "autodocsumm", + "sphinx_gallery.gen_gallery", ] autodoc_typehints = "description" -autoclass_content = 'class' -autodoc_class_signature = 'separated' -autodoc_member_order = 'alphabetical' +autoclass_content = "class" +autodoc_class_signature = "separated" +autodoc_member_order = "alphabetical" -templates_path = ['_templates'] +templates_path = ["_templates"] exclude_patterns = [] sphinx_gallery_conf = { - 'examples_dirs': ['../../examples/matmul'], - 'gallery_dirs': ['getting-started/tutorials/matmul'], - 'filename_pattern': r'.*\.py', - 'download_all_examples': True, + "examples_dirs": ["../../examples/matmul"], + "gallery_dirs": ["getting-started/tutorials/matmul"], + "filename_pattern": r".*\.py", + "download_all_examples": True, } intersphinx_mapping = { @@ -60,14 +55,14 @@ # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'sphinx_book_theme' +html_theme = "sphinx_book_theme" html_theme_options = { "repository_url": "https://github.com/NVIDIA/tilus", "use_repository_button": True, "show_navbar_depth": 1, } -html_static_path = ['_static'] +html_static_path = ["_static"] html_css_files = [ - 'custom.css', + "custom.css", ] html_permalinks_icon = "" diff --git a/python/tilus/ir/layout/mfunction/mfunction.py b/python/tilus/ir/layout/mfunction/mfunction.py index 8fc28ee8..78f14663 100644 --- a/python/tilus/ir/layout/mfunction/mfunction.py +++ b/python/tilus/ir/layout/mfunction/mfunction.py @@ -17,7 +17,7 @@ import itertools from dataclasses import dataclass from functools import cached_property -from typing import Sequence +from typing import Sequence, Union import tabulate from hidet.ir.expr import Expr @@ -25,7 +25,7 @@ from tilus.extensions.hidet.ir.utils.index_transform import index_deserialize, index_serialize from tilus.utils import prod -Int = int | Expr +Int = Union[Expr, int] @dataclass(frozen=True, eq=False) diff --git a/python/tilus/ir/layout/register_layout.py b/python/tilus/ir/layout/register_layout.py index 2393875e..35482cee 100644 --- a/python/tilus/ir/layout/register_layout.py +++ b/python/tilus/ir/layout/register_layout.py @@ -18,7 +18,7 @@ import itertools from dataclasses import dataclass from functools import cached_property -from typing import Sequence +from typing import Sequence, Union import tabulate from hidet.ir.expr import Expr @@ -28,7 +28,7 @@ from tilus.ir.layout.mfunction import MultiFunction, multi_function from tilus.ir.node import IRNode -Int = int | Expr +Int = Union[Expr, int] @dataclass(frozen=True, eq=False) diff --git a/python/tilus/ir/layout/utils/cute.py b/python/tilus/ir/layout/utils/cute.py index 2cbe0e84..6093c6b1 100644 --- a/python/tilus/ir/layout/utils/cute.py +++ b/python/tilus/ir/layout/utils/cute.py @@ -22,7 +22,7 @@ from tilus.extensions.hidet.ir.primitives.swizzle import swizzle from tilus.extensions.hidet.ir.utils.index_transform import index_deserialize -Int = Expr | int +Int = Union[Expr, int] IntTuple = Int | Sequence[Union[Int, "IntTuple"]] diff --git a/python/tilus/utils/__init__.py b/python/tilus/utils/__init__.py index 5817bf25..18ff7c49 100644 --- a/python/tilus/utils/__init__.py +++ b/python/tilus/utils/__init__.py @@ -12,11 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from hidet.utils.py import * # noqa: F401, F403 +from hidet.utils.py import gcd, initialize, is_power_of_two, lcm, prod, same_list from . import stats from .bench_utils import benchmark_func from .cache_utils import clear_cache from .multiprocess import parallel_imap, parallel_map -from .py import * # noqa: F401, F403 +from .py import cdiv, floor_log2, idiv, nbytes_from_nbits, relative_to_with_walk_up, to_snake_case from .torch_utils import dtype_from_torch, dtype_to_torch diff --git a/scripts/lint/check-copyright-header.py b/scripts/lint/check-copyright-header.py index 85dbf79e..7f0b0028 100644 --- a/scripts/lint/check-copyright-header.py +++ b/scripts/lint/check-copyright-header.py @@ -131,12 +131,9 @@ def process_directory(target_dirs, use_short_header=False, check_only=False): def main(): parser = argparse.ArgumentParser(description="Check and fix copyright headers in source files") - parser.add_argument("--check", action="store_true", - help="Only check if files have headers, don't modify them") - parser.add_argument("--fix", action="store_true", - help="Automatically add missing headers to files") - parser.add_argument("--check-and-fix", action="store_true", - help="Check and fix missing headers in files") + parser.add_argument("--check", action="store_true", help="Only check if files have headers, don't modify them") + parser.add_argument("--fix", action="store_true", help="Automatically add missing headers to files") + parser.add_argument("--check-and-fix", action="store_true", help="Check and fix missing headers in files") args = parser.parse_args() @@ -154,11 +151,13 @@ def main(): # Process directories with full license header full_total, full_need_update, full_updated, full_files_needing_update = process_directory( - TARGET_DIRS, use_short_header=False, check_only=check_only or check_and_fix) + TARGET_DIRS, use_short_header=False, check_only=check_only or check_and_fix + ) # Process directories with short license header short_total, short_need_update, short_updated, short_files_needing_update = process_directory( - SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=check_only or check_and_fix) + SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=check_only or check_and_fix + ) # Combined totals total_files = full_total + short_total @@ -184,9 +183,11 @@ def main(): # Process directories again with fix mode full_total_fix, full_need_update_fix, full_updated_fix, _ = process_directory( - TARGET_DIRS, use_short_header=False, check_only=False) + TARGET_DIRS, use_short_header=False, check_only=False + ) short_total_fix, short_need_update_fix, short_updated_fix, _ = process_directory( - SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=False) + SHORT_LICENSE_TARGET_DIRS, use_short_header=True, check_only=False + ) total_updated_fix = full_updated_fix + short_updated_fix @@ -207,5 +208,7 @@ def main(): if need_update > 0 and updated == 0: sys.exit(1) sys.exit(0) + + if __name__ == "__main__": main() From 826e9a0f102f58c983aa3b4ef49f50c983bf9d00 Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 16:52:57 -0400 Subject: [PATCH 5/9] wip Signed-off-by: Yaoyao Ding --- .github/workflows/format-and-lint.yaml | 32 +++++++++++++++++++++++--- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 1 + 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index 1009f3f8..7fe71867 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -23,7 +23,33 @@ jobs: with: python-version: '3.10' - - name: Check format and lint + - name: Cache pre-commit environments + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit-${{ runner.os }}- + + - name: Install and setup pre-commit + run: | + echo "Pre-commit version: $(pre-commit --version)" + pre-commit install-hooks + + - name: Run pre-commit checks + run: | + echo "Running all pre-commit hooks..." + pre-commit run --all-files --show-diff-on-failure + + - name: Check for uncommitted changes run: | - pip list | grep -E 'ruff|mypy' # show ruff and mypy versions - bash .github/workflows/scripts/check-format-and-lint.sh + if [[ -n $(git status --porcelain) ]]; then + echo "❌ Files were modified by pre-commit hooks. Please run 'pre-commit run --all-files' locally and commit the changes." + echo "Modified files:" + git status --porcelain + echo "Showing diff:" + git diff + exit 1 + else + echo "✅ No files were modified - all checks passed!" + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd74a4ce..e8cbc9a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,9 +5,9 @@ repos: # Ruff version rev: v0.12.3 hooks: - # Run the linter with automatic fixes - uses [tool.ruff] config from pyproject.toml + # Run the linter with fixes (CI will detect if files were modified) - id: ruff-check - name: Ruff linter (with fixes) + name: Ruff linter args: [--fix] types_or: [python, pyi, jupyter] # Run the formatter - uses [tool.ruff] config from pyproject.toml diff --git a/pyproject.toml b/pyproject.toml index 1d2718a9..55b2cdd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dev = [ "ruff==0.11.0", "mypy==1.15.0", "pytest", + "pre-commit", "types-tabulate", "types-tqdm", "pandas", From 33870a7dec8d49bf652dcecd6fea874bfb2c8f33 Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 17:04:07 -0400 Subject: [PATCH 6/9] wip Signed-off-by: Yaoyao Ding --- .github/workflows/format-and-lint.yaml | 4 +- .pre-commit-config.yaml | 6 + scripts/lint/check-commit-signature.py | 206 +++++++++++++++++++++++++ scripts/lint/format-and-lint.sh | 42 ----- scripts/{lint => }/lock-gpu-clocks.py | 0 5 files changed, 214 insertions(+), 44 deletions(-) create mode 100644 scripts/lint/check-commit-signature.py delete mode 100644 scripts/lint/format-and-lint.sh rename scripts/{lint => }/lock-gpu-clocks.py (100%) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index 7fe71867..b99239b8 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -38,8 +38,8 @@ jobs: - name: Run pre-commit checks run: | - echo "Running all pre-commit hooks..." - pre-commit run --all-files --show-diff-on-failure + echo "Running all pre-commit hooks (skipping commit signature check)..." + SKIP=check-commit-signature pre-commit run --all-files --show-diff-on-failure - name: Check for uncommitted changes run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e8cbc9a6..4be37bc1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,6 +36,12 @@ repos: language: system types: [python] pass_filenames: false + - id: check-commit-signature + name: Check commit signatures + entry: python scripts/lint/check-commit-signature.py --check + language: system + pass_filenames: false + stages: [manual] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: diff --git a/scripts/lint/check-commit-signature.py b/scripts/lint/check-commit-signature.py new file mode 100644 index 00000000..16203287 --- /dev/null +++ b/scripts/lint/check-commit-signature.py @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import subprocess +import sys +from typing import List, Tuple + + +def run_git_command(args: List[str], capture_output: bool = True) -> Tuple[int, str, str]: + """Run a git command and return exit code, stdout, stderr.""" + try: + result = subprocess.run(["git"] + args, capture_output=capture_output, text=True, check=False) + return result.returncode, result.stdout.strip(), result.stderr.strip() + except FileNotFoundError: + print("Error: git command not found") + sys.exit(1) + + +def get_merge_base(branch: str = "main") -> str: + """Get the merge base between HEAD and origin/branch.""" + # Fetch from origin first + print("Fetching from origin...") + exit_code, _, stderr = run_git_command(["fetch", "origin"]) + if exit_code != 0: + print(f"Error fetching from origin: {stderr}") + sys.exit(1) + + # Get merge base + exit_code, base_commit, stderr = run_git_command(["merge-base", "HEAD", f"origin/{branch}"]) + if exit_code != 0: + print(f"Error finding merge base with origin/{branch}: {stderr}") + sys.exit(1) + + return base_commit + + +def get_commit_range(base_commit: str) -> List[str]: + """Get list of commit hashes between base_commit and HEAD.""" + exit_code, commit_list, stderr = run_git_command(["rev-list", f"{base_commit}..HEAD"]) + if exit_code != 0: + print(f"Error getting commit range: {stderr}") + sys.exit(1) + + return commit_list.split("\n") if commit_list else [] + + +def check_commit_signature(commit_hash: str) -> bool: + """Check if a commit has a Signed-off-by line.""" + exit_code, commit_message, stderr = run_git_command(["log", "--format=%B", "-n", "1", commit_hash]) + if exit_code != 0: + print(f"Error getting commit message for {commit_hash}: {stderr}") + return False + + # Check for "Signed-off-by:" line + return "Signed-off-by:" in commit_message + + +def get_commit_info(commit_hash: str) -> str: + """Get short commit info for display.""" + exit_code, info, stderr = run_git_command(["log", "--format=%h %s", "-n", "1", commit_hash]) + if exit_code != 0: + return f"{commit_hash[:8]} (error getting info)" + return info + + +def check_commits(base_commit: str) -> Tuple[List[str], List[str]]: + """Check all commits between base and HEAD for signatures. + + Returns: + Tuple of (signed_commits, unsigned_commits) + """ + commits = get_commit_range(base_commit) + + if not commits: + print("No commits found between base and HEAD") + return [], [] + + signed_commits = [] + unsigned_commits = [] + + print(f"Checking {len(commits)} commits for signatures...") + + for commit in commits: + if check_commit_signature(commit): + signed_commits.append(commit) + else: + unsigned_commits.append(commit) + + return signed_commits, unsigned_commits + + +def fix_commit_signatures(base_commit: str) -> bool: + """Use git rebase --signoff to sign all commits.""" + print(f"Signing all commits between {base_commit[:8]} and HEAD...") + + # Run git rebase with --signoff + print("Running: git rebase --signoff") + exit_code, stdout, stderr = run_git_command(["rebase", base_commit, "--signoff"], capture_output=False) + + if exit_code == 0: + print("✅ Successfully signed all commits") + return True + else: + print(f"❌ Error during rebase: {stderr}") + print("You may need to resolve conflicts and continue the rebase manually") + return False + + +def main(): + parser = argparse.ArgumentParser(description="Check and fix commit signatures") + parser.add_argument("--check", action="store_true", help="Only check if commits are signed, don't modify them") + parser.add_argument("--fix", action="store_true", help="Sign all unsigned commits using rebase") + parser.add_argument("--branch", default="main", help="Base branch to compare against (default: main)") + parser.add_argument( + "--non-interactive", action="store_true", help="Don't prompt for confirmation when fixing (useful for CI)" + ) + + args = parser.parse_args() + + # Ensure only one of check or fix is specified + if args.check and args.fix: + parser.error("Only one of --check or --fix can be specified") + + # Default behavior is check if neither is specified + if not args.check and not args.fix: + args.check = True + + try: + # Get the merge base + base_commit = get_merge_base(args.branch) + print(f"Base commit: {base_commit[:8]}") + + if args.check: + # Check mode: report unsigned commits and exit with appropriate code + signed_commits, unsigned_commits = check_commits(base_commit) + + if signed_commits: + print(f"\n✅ Signed commits ({len(signed_commits)}):") + for commit in signed_commits: + print(f" {get_commit_info(commit)}") + + if unsigned_commits: + print(f"\n❌ Unsigned commits ({len(unsigned_commits)}):") + for commit in unsigned_commits: + print(f" {get_commit_info(commit)}") + print("\nRun with --fix to sign these commits:") + print(f" python {sys.argv[0]} --fix") + sys.exit(1) + else: + print(f"\n✅ All {len(signed_commits)} commits are properly signed!") + sys.exit(0) + + elif args.fix: + # Fix mode: sign all commits + signed_commits, unsigned_commits = check_commits(base_commit) + + if not unsigned_commits: + print("✅ All commits are already signed!") + sys.exit(0) + + print(f"\n📝 Found {len(unsigned_commits)} unsigned commits:") + for commit in unsigned_commits: + print(f" {get_commit_info(commit)}") + + # Ask for confirmation unless non-interactive mode + if not args.non_interactive: + try: + response = input(f"\nSign these {len(unsigned_commits)} commits? [y/N]: ") + if response.lower() not in ["y", "yes"]: + print("Aborted by user") + sys.exit(1) + except KeyboardInterrupt: + print("\nAborted by user") + sys.exit(1) + else: + print(f"\nRunning in non-interactive mode, proceeding to sign {len(unsigned_commits)} commits...") + + # Perform the fix + if fix_commit_signatures(base_commit): + sys.exit(0) + else: + sys.exit(1) + + except KeyboardInterrupt: + print("\nAborted by user") + sys.exit(1) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lint/format-and-lint.sh b/scripts/lint/format-and-lint.sh deleted file mode 100644 index 31f661eb..00000000 --- a/scripts/lint/format-and-lint.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Exit script on first error -set -e - -# Get the directory of the script -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# Define the target directory (relative to script location) -TARGET_DIR=$(realpath "$SCRIPT_DIR/../python") -EXAMPLE_DIR=$(realpath "$SCRIPT_DIR/../examples") -TEST_DIR=$(realpath "$SCRIPT_DIR/../tests") -SCRIPTS_DIR=$(realpath "$SCRIPT_DIR") - -# Run Ruff to format and fix issues -echo "Running ruff to clean code..." -ruff check --fix "$TARGET_DIR" "$TEST_DIR" "$EXAMPLE_DIR" "$SCRIPTS_DIR" -ruff format "$TARGET_DIR" "$TEST_DIR" "$EXAMPLE_DIR" "$SCRIPTS_DIR" - -echo "Ruff completed successfully." - -# Run Mypy for static type checking -echo "Running mypy for type checking..." -mypy "$TARGET_DIR" - -echo "mypy completed successfully." - -echo "Code cleaning process finished!" diff --git a/scripts/lint/lock-gpu-clocks.py b/scripts/lock-gpu-clocks.py similarity index 100% rename from scripts/lint/lock-gpu-clocks.py rename to scripts/lock-gpu-clocks.py From ffec9a6a55fe052d61099713d4f1f1c10664fe69 Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 17:04:53 -0400 Subject: [PATCH 7/9] empty Signed-off-by: Yaoyao Ding From 9f68e72dbcee2a7f57e9d984ea8da89b0d9818b6 Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 17:05:37 -0400 Subject: [PATCH 8/9] empty with -s Signed-off-by: Yaoyao Ding From 989642d92ca2feb0a5d799b59a455d14197048ba Mon Sep 17 00:00:00 2001 From: Yaoyao Ding Date: Fri, 31 Oct 2025 17:16:33 -0400 Subject: [PATCH 9/9] wip Signed-off-by: Yaoyao Ding --- .pre-commit-config.yaml | 6 - scripts/lint/check-commit-signature.py | 306 ++++++++++++++++++++++--- 2 files changed, 271 insertions(+), 41 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4be37bc1..e8cbc9a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,12 +36,6 @@ repos: language: system types: [python] pass_filenames: false - - id: check-commit-signature - name: Check commit signatures - entry: python scripts/lint/check-commit-signature.py --check - language: system - pass_filenames: false - stages: [manual] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: diff --git a/scripts/lint/check-commit-signature.py b/scripts/lint/check-commit-signature.py index 16203287..3bb014b2 100644 --- a/scripts/lint/check-commit-signature.py +++ b/scripts/lint/check-commit-signature.py @@ -16,7 +16,20 @@ import argparse import subprocess import sys -from typing import List, Tuple +from dataclasses import dataclass +from typing import Dict, List, Tuple + + +@dataclass +class CommitSignatureStatus: + """Status of a commit's signatures.""" + + hash: str + has_signoff: bool + has_gpg_signature: bool + gpg_status: str # "valid", "invalid", "missing", "error" + gpg_details: str + short_info: str def run_git_command(args: List[str], capture_output: bool = True) -> Tuple[int, str, str]: @@ -47,6 +60,41 @@ def get_merge_base(branch: str = "main") -> str: return base_commit +def check_working_tree_clean() -> bool: + """Check if the working tree is clean (no unstaged changes).""" + exit_code, output, stderr = run_git_command(["status", "--porcelain"]) + if exit_code != 0: + print(f"Error checking git status: {stderr}") + return False + + return len(output.strip()) == 0 + + +def prompt_for_clean_working_tree() -> None: + """Check for unstaged changes and prompt user to clean them.""" + if check_working_tree_clean(): + return # Working tree is clean, proceed + + print("\n⚠️ WORKING TREE NOT CLEAN") + print("The following files have unstaged changes:") + + # Show the status + exit_code, output, stderr = run_git_command(["status", "--porcelain"], capture_output=False) + + print("\n❌ Cannot proceed with rebase while there are unstaged changes.") + print("\n🔧 TO FIX THIS, choose one of the following:") + print(" 1. Commit your changes:") + print(" git add .") + print(' git commit -m "Your commit message"') + print(" 2. Stash your changes:") + print(" git stash") + print(" 3. Discard your changes (CAREFUL!):") + print(" git checkout -- .") + print("\n💡 Then run the script again.") + + sys.exit(1) + + def get_commit_range(base_commit: str) -> List[str]: """Get list of commit hashes between base_commit and HEAD.""" exit_code, commit_list, stderr = run_git_command(["rev-list", f"{base_commit}..HEAD"]) @@ -68,6 +116,46 @@ def check_commit_signature(commit_hash: str) -> bool: return "Signed-off-by:" in commit_message +def check_gpg_signature(commit_hash: str) -> Tuple[bool, str, str]: + """Check if a commit has a valid GPG signature. + + Returns: + Tuple of (has_signature, status, details) + status: "valid", "invalid", "missing", "error" + """ + exit_code, output, stderr = run_git_command(["verify-commit", commit_hash]) + + if exit_code == 0: + return True, "valid", output + elif "no signature found" in stderr.lower() or "bad signature" in output.lower(): + return False, "missing", stderr + elif "bad signature" in stderr.lower() or "invalid" in stderr.lower(): + return True, "invalid", stderr + else: + return False, "error", f"Unknown error: {stderr}" + + +def get_comprehensive_commit_status(commit_hash: str) -> CommitSignatureStatus: + """Get comprehensive signature status for a commit.""" + # Check sign-off + has_signoff = check_commit_signature(commit_hash) + + # Check GPG signature + has_gpg, gpg_status, gpg_details = check_gpg_signature(commit_hash) + + # Get commit info + short_info = get_commit_info(commit_hash) + + return CommitSignatureStatus( + hash=commit_hash, + has_signoff=has_signoff, + has_gpg_signature=has_gpg, + gpg_status=gpg_status, + gpg_details=gpg_details, + short_info=short_info, + ) + + def get_commit_info(commit_hash: str) -> str: """Get short commit info for display.""" exit_code, info, stderr = run_git_command(["log", "--format=%h %s", "-n", "1", commit_hash]) @@ -76,30 +164,167 @@ def get_commit_info(commit_hash: str) -> str: return info -def check_commits(base_commit: str) -> Tuple[List[str], List[str]]: +def check_commits(base_commit: str) -> Dict[str, List[CommitSignatureStatus]]: """Check all commits between base and HEAD for signatures. Returns: - Tuple of (signed_commits, unsigned_commits) + Dict with keys: "all_good", "missing_signoff", "missing_gpg", "invalid_gpg", "missing_both" """ commits = get_commit_range(base_commit) if not commits: print("No commits found between base and HEAD") - return [], [] - - signed_commits = [] - unsigned_commits = [] + return {"all_good": [], "missing_signoff": [], "missing_gpg": [], "invalid_gpg": [], "missing_both": []} print(f"Checking {len(commits)} commits for signatures...") + # Categorize commits + all_good = [] + missing_signoff = [] + missing_gpg = [] + invalid_gpg = [] + missing_both = [] + for commit in commits: - if check_commit_signature(commit): - signed_commits.append(commit) + status = get_comprehensive_commit_status(commit) + + # Categorize based on status + if status.has_signoff and status.gpg_status == "valid": + all_good.append(status) + elif not status.has_signoff and status.gpg_status in ["missing", "error"]: + missing_both.append(status) + elif not status.has_signoff: + missing_signoff.append(status) + elif status.gpg_status in ["missing", "error"]: + missing_gpg.append(status) + elif status.gpg_status == "invalid": + invalid_gpg.append(status) else: - unsigned_commits.append(commit) + # Edge case - has signoff but something else is wrong with GPG + if status.has_signoff: + missing_gpg.append(status) + else: + missing_signoff.append(status) + + return { + "all_good": all_good, + "missing_signoff": missing_signoff, + "missing_gpg": missing_gpg, + "invalid_gpg": invalid_gpg, + "missing_both": missing_both, + } + + +def print_signature_report(results: Dict[str, List[CommitSignatureStatus]]) -> None: + """Print a comprehensive human-readable report of signature status.""" + total_commits = sum(len(commits) for commits in results.values()) + + print(f"\n{'=' * 60}") + print("COMMIT SIGNATURE ANALYSIS REPORT") + print(f"{'=' * 60}") + print(f"Total commits analyzed: {total_commits}") + + # Print good commits + if results["all_good"]: + print(f"\n✅ FULLY SIGNED COMMITS ({len(results['all_good'])}):") + print(" These commits have both DCO sign-off and valid GPG signatures") + for status in results["all_good"]: + print(f" {status.short_info}") + + # Print issues with clear explanations and todos + issues_found = False + + if results["missing_both"]: + issues_found = True + print(f"\n❌ MISSING BOTH SIGNATURES ({len(results['missing_both'])}):") + print(" ⚠️ CRITICAL: These commits lack both DCO sign-off AND GPG signatures") + print(" 📋 TODO: Add both sign-off and GPG signature") + for status in results["missing_both"]: + print(f" {status.short_info}") + print("\n 🔧 HOW TO FIX:") + print(" 1. For DCO sign-off: Run the script with --fix") + print(" 2. For GPG signatures: git rebase --gpg-sign ") + print(" 3. Or configure: git config commit.gpgsign true") + + if results["missing_signoff"]: + issues_found = True + print(f"\n📝 MISSING DCO SIGN-OFF ({len(results['missing_signoff'])}):") + print(" ⚠️ These commits lack 'Signed-off-by:' lines (Developer Certificate of Origin)") + print(" 📋 TODO: Add DCO sign-off to comply with contribution guidelines") + for status in results["missing_signoff"]: + print(f" {status.short_info}") + print("\n 🔧 HOW TO FIX:") + print(" Run: python scripts/lint/check-commit-signature.py --fix") + print(" Or manually: git rebase --signoff ") + + if results["missing_gpg"]: + issues_found = True + print(f"\n🔐 MISSING GPG SIGNATURES ({len(results['missing_gpg'])}):") + print(" ⚠️ These commits are not cryptographically signed") + print(" 📋 TODO: Add GPG signatures for authenticity verification") + for status in results["missing_gpg"]: + print(f" {status.short_info}") + print("\n 🔧 HOW TO FIX:") + print(" 1. Set up GPG key: gpg --full-generate-key") + print(" 2. Configure git: git config user.signingkey ") + print(" 3. Re-sign commits: git rebase --gpg-sign ") + print(" 4. Or enable by default: git config commit.gpgsign true") + + if results["invalid_gpg"]: + issues_found = True + print(f"\n🚫 INVALID GPG SIGNATURES ({len(results['invalid_gpg'])}):") + print(" ⚠️ SECURITY WARNING: These commits have invalid or corrupted signatures") + print(" 📋 TODO: Investigate and re-sign these commits") + for status in results["invalid_gpg"]: + print(f" {status.short_info}") + print(f" Error: {status.gpg_details}") + print("\n 🔧 HOW TO FIX:") + print(" 1. Verify your GPG key is valid: gpg --list-secret-keys") + print(" 2. Re-sign the commits: git rebase --gpg-sign ") + print(" 3. If key is compromised, revoke and create new key") + + # Summary and recommendations + print(f"\n{'=' * 60}") + if not issues_found: + print("🎉 EXCELLENT! All commits are properly signed with both DCO and GPG signatures.") + print(" Your commits meet the highest security and compliance standards.") + else: + print("📊 SUMMARY OF ISSUES:") + if results["missing_both"]: + print(f" • {len(results['missing_both'])} commits missing BOTH signatures (highest priority)") + if results["missing_signoff"]: + print(f" • {len(results['missing_signoff'])} commits missing DCO sign-off") + if results["missing_gpg"]: + print(f" • {len(results['missing_gpg'])} commits missing GPG signatures") + if results["invalid_gpg"]: + print(f" • {len(results['invalid_gpg'])} commits with INVALID GPG signatures") + + print("\n🎯 QUICK ACTION PLAN:") + print(" 1. Run with --fix to add missing DCO sign-offs") + print(" 2. Set up GPG signing: git config commit.gpgsign true") + print(" 3. Re-sign commits: git rebase --gpg-sign ") + + print(f"{'=' * 60}") + + +def get_fix_summary(results: Dict[str, List[CommitSignatureStatus]]) -> str: + """Generate a summary of what the fix operation will do.""" + commits_needing_signoff = len(results["missing_signoff"]) + len(results["missing_both"]) - return signed_commits, unsigned_commits + if commits_needing_signoff == 0: + return "✅ All commits already have DCO sign-off!" + + summary = f"📝 Will add DCO sign-off to {commits_needing_signoff} commits:\n" + + for status in results["missing_signoff"] + results["missing_both"]: + summary += f" • {status.short_info}\n" + + if results["missing_gpg"] or results["invalid_gpg"]: + gpg_count = len(results["missing_gpg"]) + len(results["invalid_gpg"]) + summary += f"\n🔐 Note: {gpg_count} commits will still need GPG signatures after this fix.\n" + summary += " Use 'git rebase --gpg-sign ' to add GPG signatures.\n" + + return summary def fix_commit_signatures(base_commit: str) -> bool: @@ -144,41 +369,52 @@ def main(): print(f"Base commit: {base_commit[:8]}") if args.check: - # Check mode: report unsigned commits and exit with appropriate code - signed_commits, unsigned_commits = check_commits(base_commit) - - if signed_commits: - print(f"\n✅ Signed commits ({len(signed_commits)}):") - for commit in signed_commits: - print(f" {get_commit_info(commit)}") - - if unsigned_commits: - print(f"\n❌ Unsigned commits ({len(unsigned_commits)}):") - for commit in unsigned_commits: - print(f" {get_commit_info(commit)}") - print("\nRun with --fix to sign these commits:") - print(f" python {sys.argv[0]} --fix") + # Check mode: report signature status and exit with appropriate code + results = check_commits(base_commit) + + # Print comprehensive report + print_signature_report(results) + + # Exit with error if any issues found + issues_found = any(len(commits) > 0 for key, commits in results.items() if key != "all_good") + + if issues_found: + print("\n❌ Issues found with commit signatures.") + print(f"💡 Run 'python {sys.argv[0]} --fix' to fix DCO sign-off issues.") sys.exit(1) else: - print(f"\n✅ All {len(signed_commits)} commits are properly signed!") + print("\n✅ All commits have proper signatures!") sys.exit(0) elif args.fix: - # Fix mode: sign all commits - signed_commits, unsigned_commits = check_commits(base_commit) + # Fix mode: only fix DCO sign-off (GPG signatures require separate handling) + + # Check for unstaged changes before proceeding + print("Checking working tree status...") + prompt_for_clean_working_tree() + + results = check_commits(base_commit) + + commits_needing_signoff = len(results["missing_signoff"]) + len(results["missing_both"]) + + if commits_needing_signoff == 0: + print("✅ All commits already have DCO sign-off!") + + # Check if only GPG issues remain + gpg_issues = len(results["missing_gpg"]) + len(results["invalid_gpg"]) + if gpg_issues > 0: + print(f"\n🔐 Note: {gpg_issues} commits still need GPG signatures.") + print(" Use 'git rebase --gpg-sign ' to add GPG signatures.") - if not unsigned_commits: - print("✅ All commits are already signed!") sys.exit(0) - print(f"\n📝 Found {len(unsigned_commits)} unsigned commits:") - for commit in unsigned_commits: - print(f" {get_commit_info(commit)}") + # Show what will be fixed + print(get_fix_summary(results)) # Ask for confirmation unless non-interactive mode if not args.non_interactive: try: - response = input(f"\nSign these {len(unsigned_commits)} commits? [y/N]: ") + response = input(f"\nAdd DCO sign-off to {commits_needing_signoff} commits? [y/N]: ") if response.lower() not in ["y", "yes"]: print("Aborted by user") sys.exit(1) @@ -186,7 +422,7 @@ def main(): print("\nAborted by user") sys.exit(1) else: - print(f"\nRunning in non-interactive mode, proceeding to sign {len(unsigned_commits)} commits...") + print(f"\nRunning in non-interactive mode, proceeding to sign {commits_needing_signoff} commits...") # Perform the fix if fix_commit_signatures(base_commit):