From dd25255ddcae7822f5f7895b84cd32f0386c9a8b Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Wed, 19 Nov 2025 10:33:58 -0800 Subject: [PATCH 1/9] [Examples][CI] Enable testing of examples through lit Adds lit as a test dependency via pyproject.toml. Adds FileCheck as an external dependency -- for CI we use Ubuntu's llvm-dev package. --- .github/workflows/examples.yml | 28 ++++++++----------- lit.cfg.py | 16 +++++++++++ pyproject.toml | 6 ++++ python/examples/ingress/torch/lit.local.cfg | 1 + .../examples/ingress/torch/mlp_from_file.py | 2 ++ .../examples/ingress/torch/mlp_from_model.py | 2 ++ python/examples/lit.local.cfg | 1 + python/examples/mlir/compile_and_run.py | 2 ++ ...sform_a_payload_according_to_a_schedule.py | 15 ++++++++-- 9 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 lit.cfg.py create mode 100644 python/examples/ingress/torch/lit.local.cfg create mode 100644 python/examples/lit.local.cfg diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 991d947..74ef6e2 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -7,7 +7,11 @@ on: jobs: Examples: - runs-on: ubuntu-latest + strategy: + matrix: + arch: [x86_64, AArch64] + + runs-on: ${{ matrix.arch == 'AArch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v5 @@ -16,21 +20,11 @@ jobs: - name: Install the project and its dependencies run: | - uv sync - uv sync --extra ingress-torch-cpu - - - name: Run MLP From File - run: |- - uv run python/examples/ingress/torch/mlp_from_file.py + sudo apt-get install llvm-dev # Obtain FileCheck, used in testing. + uv sync --extra ingress-torch-cpu --group test - - name: Run MLP From Module - run: |- - uv run python/examples/ingress/torch/mlp_from_model.py - - - name: Run Compile And Run - run: |- - uv run python/examples/mlir/compile_and_run.py + - name: Run lit-enabled examples as tests + run: | + export FILECHECKPATH=FileCheck-18 # Ubuntu's llvm-dev appends a version number. + uv run lit python/examples # Makes sure to substitute FileCheck for $FILECHECKPATH - - name: Run apply basic schedule to basic payload - run: |- - uv run python/examples/schedule/transform_a_payload_according_to_a_schedule.py diff --git a/lit.cfg.py b/lit.cfg.py new file mode 100644 index 0000000..7d92513 --- /dev/null +++ b/lit.cfg.py @@ -0,0 +1,16 @@ +import os + +import lit.formats +from lit.TestingConfig import TestingConfig + +# Imagine that, all your variables defined and with types! +assert isinstance(config := eval("config"), TestingConfig) + +config.name = "Lighthouse test suite" +config.test_format = lit.formats.ShTest(True) +config.test_source_root = os.path.dirname(__file__) +config.test_exec_root = os.path.dirname(__file__) + "/lit.out" + +config.substitutions.append(("%PYTHON", "uv run")) +if filecheck_path := os.environ.get("FILECHECKPATH"): + config.substitutions.append(("FileCheck", filecheck_path)) diff --git a/pyproject.toml b/pyproject.toml index e559632..912a34d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,11 @@ dependencies = [ "mlir-python-bindings==20251118+99630eb1b" ] +[dependency-groups] +test = [ + "lit==18.1.8" # Tool to configure, discover and run tests +] + [project.optional-dependencies] ingress_torch_mlir = [ "torch-mlir==20251122.639" @@ -66,6 +71,7 @@ name = "torch_mlir" url = "https://github.com/llvm/torch-mlir-release/releases/expanded_assets/dev-wheels" explicit = true +# Derive package version from version in code [build-system] requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" diff --git a/python/examples/ingress/torch/lit.local.cfg b/python/examples/ingress/torch/lit.local.cfg new file mode 100644 index 0000000..f53a309 --- /dev/null +++ b/python/examples/ingress/torch/lit.local.cfg @@ -0,0 +1 @@ +config.excludes = ["MLPModel"] \ No newline at end of file diff --git a/python/examples/ingress/torch/mlp_from_file.py b/python/examples/ingress/torch/mlp_from_file.py index 01edeec..b8afc36 100644 --- a/python/examples/ingress/torch/mlp_from_file.py +++ b/python/examples/ingress/torch/mlp_from_file.py @@ -1,3 +1,5 @@ +# RUN: %PYTHON %s + """ Example demonstrating how to load a PyTorch model to MLIR using Lighthouse without initializing the model class on the user's side. diff --git a/python/examples/ingress/torch/mlp_from_model.py b/python/examples/ingress/torch/mlp_from_model.py index b51af80..6621e90 100644 --- a/python/examples/ingress/torch/mlp_from_model.py +++ b/python/examples/ingress/torch/mlp_from_model.py @@ -1,3 +1,5 @@ +# RUN: %PYTHON %s + """ Example demonstrating how to load an already initialized PyTorch model to MLIR using Lighthouse. diff --git a/python/examples/lit.local.cfg b/python/examples/lit.local.cfg new file mode 100644 index 0000000..a9e7d1a --- /dev/null +++ b/python/examples/lit.local.cfg @@ -0,0 +1 @@ +config.suffixes = {'.py'} \ No newline at end of file diff --git a/python/examples/mlir/compile_and_run.py b/python/examples/mlir/compile_and_run.py index 009d9d5..0d8b8f1 100644 --- a/python/examples/mlir/compile_and_run.py +++ b/python/examples/mlir/compile_and_run.py @@ -1,3 +1,5 @@ +# RUN: %PYTHON %s + import torch import argparse diff --git a/python/examples/schedule/transform_a_payload_according_to_a_schedule.py b/python/examples/schedule/transform_a_payload_according_to_a_schedule.py index 12702ab..fcbbe7b 100644 --- a/python/examples/schedule/transform_a_payload_according_to_a_schedule.py +++ b/python/examples/schedule/transform_a_payload_according_to_a_schedule.py @@ -1,3 +1,5 @@ +# RUN: %PYTHON %s | FileCheck %s + # Simply demonstrates applying a schedule to a payload. # To do so generates a basic payload and a basic schedule, purely as an example. @@ -24,16 +26,23 @@ def example_payload() -> Module: with InsertionPoint(payload.body): matrixType = RankedTensorType.get([16, 16], F32Type.get()) + # NB: Do the CHECKing on the transformed output: + # CHECK-LABEL: result of applying schedule to payload + # CHECK: func.func @fold_add_on_two_matmuls + # CHECK-SAME: (%[[MATRIX_A:.*]]: {{.*}}, %[[MATRIX_B:.*]]: {{.*}}, %[[WEIGHTS:.*]]: {{.*}}) @func.func(matrixType, matrixType, matrixType) def fold_add_on_two_matmuls(matrixA, matrixB, weights): empty = tensor.empty(matrixType.shape, matrixType.element_type) c0 = arith.constant(F32Type.get(), 0.0) + # CHECK: %[[ZERO_INIT:.*]] = linalg.fill zero_init = linalg.fill(c0, outs=[empty]) + # CHECK: %[[A_X_WEIGHTS:.*]] = linalg.matmul ins(%[[MATRIX_A]], %[[WEIGHTS]]{{.*}}) outs(%[[ZERO_INIT]] A_x_weights = linalg.matmul(matrixA, weights, outs=[zero_init]) - empty2 = tensor.empty(matrixType.shape, matrixType.element_type) - zero_init2 = linalg.fill(c0, outs=[empty2]) - B_x_weights = linalg.matmul(matrixB, weights, outs=[zero_init2]) + # CHECK: %[[RES:.*]] = linalg.matmul ins(%[[MATRIX_B]], %[[WEIGHTS]]{{.*}}) outs(%[[A_X_WEIGHTS]] + B_x_weights = linalg.matmul(matrixB, weights, outs=[zero_init]) + # CHECK-NOT: linalg.add added = linalg.add(A_x_weights, B_x_weights, outs=[empty]) + # CHECK: return %[[RES]] return added print(payload) From 28c43f4065afc7d5093dac502b63e4793b8cfb3f Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Sat, 22 Nov 2025 07:14:22 -0800 Subject: [PATCH 2/9] Newline --- python/examples/ingress/torch/lit.local.cfg | 3 ++- python/examples/lit.local.cfg | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/examples/ingress/torch/lit.local.cfg b/python/examples/ingress/torch/lit.local.cfg index f53a309..4678a86 100644 --- a/python/examples/ingress/torch/lit.local.cfg +++ b/python/examples/ingress/torch/lit.local.cfg @@ -1 +1,2 @@ -config.excludes = ["MLPModel"] \ No newline at end of file +config.excludes = ["MLPModel"] + diff --git a/python/examples/lit.local.cfg b/python/examples/lit.local.cfg index a9e7d1a..a6d9e82 100644 --- a/python/examples/lit.local.cfg +++ b/python/examples/lit.local.cfg @@ -1 +1,2 @@ -config.suffixes = {'.py'} \ No newline at end of file +config.suffixes = {'.py'} + From c6630bcee2027f9986fe8fd52598bb131bb809ae Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Sat, 22 Nov 2025 07:17:11 -0800 Subject: [PATCH 3/9] -y --- .github/workflows/examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 74ef6e2..0ef6546 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -20,7 +20,7 @@ jobs: - name: Install the project and its dependencies run: | - sudo apt-get install llvm-dev # Obtain FileCheck, used in testing. + sudo apt-get install -y llvm-dev # Obtain FileCheck, used in testing. uv sync --extra ingress-torch-cpu --group test - name: Run lit-enabled examples as tests From dca70f7e22471498015584eb02f601b37bfc9e8b Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Sat, 22 Nov 2025 07:29:55 -0800 Subject: [PATCH 4/9] Docs --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index d09a4f8..d4f4454 100644 --- a/README.md +++ b/README.md @@ -117,3 +117,10 @@ pip install .[ingress_torch_cpu] \ --extra-index-url https://download.pytorch.org/whl \ --only-binary :all: ``` + +## Running tests + +Running the tests is as simple as `lit .` in the root of the project. + +We assume that the [`FileCheck`](https://llvm.org/docs/CommandGuide/FileCheck.html) and [`lit`](https://llvm.org/docs/CommandGuide/lit.html) executables are available on the `PATH`. +To obtain the [Python package for `lit`](https://pypi.org/project/lit/), simply run `uv sync --group test`. From 988e0622d5de247007ed41025b31c117b85d520e Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Sun, 23 Nov 2025 03:48:21 -0800 Subject: [PATCH 5/9] FILECHECKPATH -> FILECHECK and docs for setting this env var --- .github/workflows/examples.yml | 3 +-- README.md | 7 +++++++ lit.cfg.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 0ef6546..b32703d 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -25,6 +25,5 @@ jobs: - name: Run lit-enabled examples as tests run: | - export FILECHECKPATH=FileCheck-18 # Ubuntu's llvm-dev appends a version number. + export FILECHECK=FileCheck-18 # Ubuntu's llvm-dev appends a version number. uv run lit python/examples # Makes sure to substitute FileCheck for $FILECHECKPATH - diff --git a/README.md b/README.md index d4f4454..73766f8 100644 --- a/README.md +++ b/README.md @@ -123,4 +123,11 @@ pip install .[ingress_torch_cpu] \ Running the tests is as simple as `lit .` in the root of the project. We assume that the [`FileCheck`](https://llvm.org/docs/CommandGuide/FileCheck.html) and [`lit`](https://llvm.org/docs/CommandGuide/lit.html) executables are available on the `PATH`. + +
+ +Obtaining `FileCheck` and `lit`. + To obtain the [Python package for `lit`](https://pypi.org/project/lit/), simply run `uv sync --group test`. +In case the `FileCheck` executable happens to be available under a different name/location, e.g. as `FileCheck-18` from Ubuntu's `llvm-dev` package, set the `FILECHECK` environment variable when invoking `lit`. +
diff --git a/lit.cfg.py b/lit.cfg.py index 7d92513..e8b7547 100644 --- a/lit.cfg.py +++ b/lit.cfg.py @@ -12,5 +12,5 @@ config.test_exec_root = os.path.dirname(__file__) + "/lit.out" config.substitutions.append(("%PYTHON", "uv run")) -if filecheck_path := os.environ.get("FILECHECKPATH"): +if filecheck_path := os.environ.get("FILECHECK"): config.substitutions.append(("FileCheck", filecheck_path)) From d0d4e454e37a2009407d4d96797ac05b5abe384b Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Sun, 23 Nov 2025 05:27:46 -0800 Subject: [PATCH 6/9] Hide details in docs by default and remove extra newlines --- README.md | 6 +++--- python/examples/ingress/torch/lit.local.cfg | 1 - python/examples/lit.local.cfg | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 73766f8..41bfe18 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,8 @@ We assume that the [`FileCheck`](https://llvm.org/docs/CommandGuide/FileCheck.ht
-Obtaining `FileCheck` and `lit`. +Obtaining FileCheck and lit. -To obtain the [Python package for `lit`](https://pypi.org/project/lit/), simply run `uv sync --group test`. -In case the `FileCheck` executable happens to be available under a different name/location, e.g. as `FileCheck-18` from Ubuntu's `llvm-dev` package, set the `FILECHECK` environment variable when invoking `lit`. +To obtain the Python package for lit, simply run uv sync --group test. +In case the FileCheck executable happens to be available under a different name/location, e.g. as FileCheck-18 from Ubuntu's llvm-dev package, set the FILECHECK environment variable when invoking lit.
diff --git a/python/examples/ingress/torch/lit.local.cfg b/python/examples/ingress/torch/lit.local.cfg index 4678a86..2813a1b 100644 --- a/python/examples/ingress/torch/lit.local.cfg +++ b/python/examples/ingress/torch/lit.local.cfg @@ -1,2 +1 @@ config.excludes = ["MLPModel"] - diff --git a/python/examples/lit.local.cfg b/python/examples/lit.local.cfg index a6d9e82..73171b0 100644 --- a/python/examples/lit.local.cfg +++ b/python/examples/lit.local.cfg @@ -1,2 +1 @@ config.suffixes = {'.py'} - From a784515f10860d84c3c3ad2dbc0b55c6bcf75fc5 Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Sun, 23 Nov 2025 05:57:20 -0800 Subject: [PATCH 7/9] Only trigger CI on push to main https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#running-your-workflow-only-when-a-push-to-specific-branches-occurs --- .github/workflows/examples.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b32703d..f5fb289 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -3,6 +3,8 @@ name: Lighthouse Examples on: workflow_dispatch: push: + branches: + - 'main' pull_request: jobs: From 9fca8a3859f03e6719f3f9833b4b72f2331c3bfb Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Sun, 23 Nov 2025 05:59:18 -0800 Subject: [PATCH 8/9] Replace old var name --- .github/workflows/examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index f5fb289..136b848 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -28,4 +28,4 @@ jobs: - name: Run lit-enabled examples as tests run: | export FILECHECK=FileCheck-18 # Ubuntu's llvm-dev appends a version number. - uv run lit python/examples # Makes sure to substitute FileCheck for $FILECHECKPATH + uv run lit python/examples # Makes sure to substitute FileCheck for $FILECHECK From 2f3e92f88ebb98ddddd29239b3aacc8d07e9ef28 Mon Sep 17 00:00:00 2001 From: Rolf Morel Date: Mon, 24 Nov 2025 03:29:09 -0800 Subject: [PATCH 9/9] From test group to dev group --- .github/workflows/examples.yml | 2 +- README.md | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 136b848..3396fb1 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -23,7 +23,7 @@ jobs: - name: Install the project and its dependencies run: | sudo apt-get install -y llvm-dev # Obtain FileCheck, used in testing. - uv sync --extra ingress-torch-cpu --group test + uv sync --extra ingress-torch-cpu - name: Run lit-enabled examples as tests run: | diff --git a/README.md b/README.md index 41bfe18..59e8f8c 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,6 @@ We assume that the [`FileCheck`](https://llvm.org/docs/CommandGuide/FileCheck.ht Obtaining FileCheck and lit. -To obtain the Python package for lit, simply run uv sync --group test. +To obtain the Python package for lit, simply run uv sync (lit is included in the "dev" dependency group). In case the FileCheck executable happens to be available under a different name/location, e.g. as FileCheck-18 from Ubuntu's llvm-dev package, set the FILECHECK environment variable when invoking lit. diff --git a/pyproject.toml b/pyproject.toml index 912a34d..9b4b2e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ dependencies = [ ] [dependency-groups] -test = [ +dev = [ "lit==18.1.8" # Tool to configure, discover and run tests ]