Improve scripts and instructions for macOS & various minor fixes (#363)

List of changes:
- `tensorflow-cpu` doesn't exist for macOS, use regular `tensorflow`
instead in `requirements.txt`
- add instructions to the documentation for building the docs
- fix the default mhlo build directory not being ignored by git (now
`bazel-build`, as it has to be a directory already present in the
upstream `.gitignore`)
- fix an AutoGraph lit test which produces slightly different output
with different TF versions when assigning tuples
- update formatting exclusions (e.g. with Enzyme directories)
- fix test that didn't clean up after itself
- update wheel build recipe with cleanup
- fix extra header in the Catalyst documentation landing page
- add other optional test dependencies to `requirements.txt`
- update installation guide with additional information on building
backends and running our various test suites

Otherwise I can confirm that the current build from source instructions
work for macOS, as well as all the default Make recipes (build, tests,
docs, formatting, demos, wheel). Also checked additional devices and
tests (Kokkos, openqasm).
Sanitized builds are left for a follow-up.

.github/workflows/build-wheel-linux-x86_64.yaml

@@ -254,15 +254,16 @@ jobs:
       run: |
         cmake -S runtime -B runtime-build -G Ninja \
               -DCMAKE_BUILD_TYPE=Release \
-              -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=$PWD/runtime-build/lib \
+              -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=$GITHUB_WORKSPACE/runtime-build/lib \
               -DPYTHON_EXECUTABLE=$(which python${{ matrix.python_version }}) \
               -Dpybind11_DIR=$(python${{ matrix.python_version }} -c "import pybind11; print(pybind11.get_cmake_dir())") \
               -DENABLE_LIGHTNING_KOKKOS=ON \
               -DLIGHTNING_GIT_TAG="latest_release" \
               -DKokkos_ENABLE_SERIAL=ON \
               -DKokkos_ENABLE_OPENMP=ON \
               -DENABLE_WARNINGS=OFF \
-              -DENABLE_OPENQASM=ON
+              -DENABLE_OPENQASM=ON \
+              -DENABLE_OPENMP=ON
 
         cmake --build runtime-build --target rt_capi
 

.github/workflows/build-wheel-macos-arm64.yaml

@@ -159,7 +159,7 @@ jobs:
 
         cmake --build enzyme-build --target EnzymeStatic-18
 
-  catalyst-macos-wheels-x86-64:
+  catalyst-macos-wheels-arm64:
     needs: [constants, build-dependencies]
     strategy:
       fail-fast: false
@@ -244,9 +244,11 @@ jobs:
               -DENABLE_LIGHTNING_KOKKOS=ON \
               -DLIGHTNING_GIT_TAG="latest_release" \
               -DKokkos_ENABLE_SERIAL=ON \
+              -DKokkos_ENABLE_OPENMP=OFF \
               -DKokkos_ENABLE_COMPLEX_ALIGN=OFF \
               -DENABLE_WARNINGS=OFF \
-              -DENABLE_OPENQASM=ON
+              -DENABLE_OPENQASM=ON \
+              -ENABLE_OPENMP=OFF
 
         cmake --build runtime-build --target rt_capi
 

.github/workflows/build-wheel-macos-x86_64.yaml

@@ -246,10 +246,12 @@ jobs:
               -DLIGHTNING_GIT_TAG="latest_release" \
               -DENABLE_LIGHTNING_KOKKOS=ON \
               -DKokkos_ENABLE_SERIAL=ON \
+              -DKokkos_ENABLE_OPENMP=OFF \
               -DKokkos_ENABLE_COMPLEX_ALIGN=OFF \
               -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF \
               -DENABLE_WARNINGS=OFF \
-              -DENABLE_OPENQASM=ON
+              -DENABLE_OPENQASM=ON \
+              -DENABLE_OPENMP=OFF
 
         cmake --build runtime-build --target rt_capi
 

Makefile

@@ -8,7 +8,7 @@ BLACKVERSIONMINOR := $(if $(BLACKVERSIONMINOR),$(BLACKVERSIONMINOR),0)
 MK_ABSPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 MK_DIR := $(dir $(MK_ABSPATH))
 LLVM_BUILD_DIR ?= $(MK_DIR)/mlir/llvm-project/build
-MHLO_BUILD_DIR ?= $(MK_DIR)/mlir/mlir-hlo/mhlo-build
+MHLO_BUILD_DIR ?= $(MK_DIR)/mlir/mlir-hlo/bazel-build
 DIALECTS_BUILD_DIR ?= $(MK_DIR)/mlir/build
 RT_BUILD_DIR ?= $(MK_DIR)/runtime/build
 ENZYME_BUILD_DIR ?= $(MK_DIR)/mlir/Enzyme/build
@@ -129,6 +129,8 @@ wheel:
 
 	$(PYTHON) $(MK_DIR)/setup.py bdist_wheel
 
+	rm -r $(MK_DIR)/build
+
 .PHONY: clean clean-all
 clean:
 	@echo "uninstall catalyst and delete all temporary and cache files"

doc/dev/installation.rst

@@ -10,6 +10,14 @@ pre-built binaries are being distributed via the Python Package Index (PyPI) for
 
     pip install pennylane-catalyst
 
+.. warning::
+
+    macOS does not ship with a system compiler by default, which Catalyst depends on. Please
+    ensure that `XCode <https://developer.apple.com/xcode/resources/`_ or the
+    ``XCode Command Line Tools`` are installed on your system before using Catalyst.
+
+    The easiest method of installation is to run ``xcode-select --install`` from the Terminal
+    app.
 
 Pre-built packages for Windows are not yet available, and comptability with other platforms is
 untested and cannot be guaranteed. If you are using one of these platforms, please
@@ -144,28 +152,37 @@ directory:
 
   make all
 
-To build each component one by one starting from the runtime, you can follow the instructions below.
+To build each component one by one starting from the runtime, or to build additional backend devices
+beyond ``lightning.qubit``, please follow the instructions below.
 
 Runtime
 """""""
 
-By default, the runtime is backed by `PennyLane-Lightning
-<https://github.com/PennyLaneAI/pennylane-lightning>`_
-requiring the use of C++20 standard library headers, and leverages the `QIR
-standard library <https://github.com/qir-alliance/qir-runner>`_. Assuming
-``libomp-dev`` is available, you can build the runtime from the top level
-directory:
+By default, the runtime builds and installs the `PennyLane-Lightning
+<https://github.com/PennyLaneAI/pennylane-lightning>`_ simulator device, which requires C++20
+standard library features. Older C++ compilers may not support this, so it is recommended to use a
+modern compiler with these features. An additional dependency, the `QIR
+standard library <https://github.com/qir-alliance/qir-runner>`_, is automatically fetched and
+built on supported platforms.
+
+From the root project directory, the runtime can then be built as follows:
 
 .. code-block:: console
 
   make runtime
 
-The runtime supports multiple backend devices, enabling the execution of quantum
-circuits locally on CPUs and GPUs, and remotely on Amazon Braket NISQ hardware.
-A list of supported backends, along with Make arguments for each device, is available in the
+Additional devices are constantly added, enabling the execution of quantum circuits on CPUs, GPUs,
+and remote services, such as Amazon Braket. The full list of supported backends, and additional
+configuration options, are available in the
 `Catalyst Runtime <https://docs.pennylane.ai/projects/catalyst/en/latest/modules/runtime.html>`_
 page.
 
+To install Catalyst with all available backends, simply run:
+
+.. code-block:: console
+
+  make runtime ENABLE_LIGHTNING_KOKKOS=ON ENABLE_OPENQASM=ON
+
 MLIR Dialects
 """""""""""""
 
@@ -234,11 +251,69 @@ To make required tools in ``llvm-project/build``, ``mlir-hlo/mhlo-build``, and
 Tests
 ^^^^^
 
-The following target runs all available test suites in Catalyst:
+The following target runs all available test suites with the default execution device in Catalyst:
 
 .. code-block:: console
 
   make test
 
 You can also test each module separately by using running the ``test-frontend``,
-``test-dialects``, and ``test-runtime`` targets instead.
+``test-dialects``, and ``test-runtime`` targets instead. Jupyter Notebook demos are also testable
+via ``test-demos``.
+
+Additional Device Backends
+""""""""""""""""""""""""""
+
+The **runtime tests** can be run on additional devices via the same flags that were used to build
+them, but using the ``test-runtime`` target instead:
+
+.. code-block:: console
+
+  make test-runtime ENABLE_LIGHTNING_KOKKOS=ON ENABLE_OPENQASM=ON
+
+.. Note::
+
+  The ``test-runtime`` targets rebuilds the runtime with the specified flags. Therefore,
+  running ``make runtime OPENQASM=ON`` and ``make test-runtime`` in succession will leave you
+  without the OpenQASM device installed.
+  In case of errors it can also help to delete the build directory.
+
+The **Python test suite** is also set up to run with different device backends. Assuming the
+respective device is available & compatible, they can be tested individually by specifying the
+PennyLane plugin device name in the test command:
+
+.. code-block:: console
+
+  make pytest TEST_BACKEND="lightning.kokkos"
+
+AWS Braket devices have their own set of tests, which can be run either locally (``LOCAL``) or on
+the AWS Braket service (``REMOTE``) as follows:
+
+.. code-block:: console
+
+  make pytest TEST_BRAKET=LOCAL
+
+Documentation
+^^^^^^^^^^^^^
+
+To build and test documentation for Catalyst, you will need to install
+`sphinx <https://www.sphinx-doc.org>`_ and other packages listed in ``doc/requirements.txt``:
+
+.. code-block:: console
+
+  pip install -r doc/requirements.txt
+
+Additionally, `doxygen <https://www.doxygen.nl>`_ is required to build C++ documentation, and
+`pandoc <https://pandoc.org>`_ to render Jupyter Notebooks.
+
+On **Debian/Ubuntu**, they can be installed via:
+
+.. code-block:: console
+
+  sudo apt install doxygen pandoc
+
+On **macOS**, `homebrew <https://brew.sh>`_ is the easiest way to install these packages:
+
+.. code-block:: console
+
+  brew install doxygen pandoc

doc/index.rst

@@ -54,7 +54,7 @@ Catalyst
 
 .. mdinclude:: ../README.md
   :start-line: 19
-  :end-line: 73
+  :end-line: 72
 
 .. toctree::
    :maxdepth: 2

frontend/test/lit/test_autograph.py

@@ -364,7 +364,7 @@ def if_assign_existing_partial_no_type_mismatch(x: float):
 def if_assign_multiple(x: float):
     """Test a conditional that assigns to multiple existing variables."""
 
-    # CHECK:   (y, z) = (0, False)
+    # CHECK:   {{\(?}}y, z{{\)?}} = (0, False)
     y, z = 0, False
 
     # CHECK:       def if_body():

frontend/test/pytest/test_compiler.py

@@ -340,6 +340,7 @@ def circuit():
         assert "PipelineA" not in e.value.args[0]
         assert "Trace" not in e.value.args[0]
         assert isfile(os.path.join(str(compiled.workspace), "2_PipelineB_FAILED.mlir"))
+        compiled.workspace.cleanup()
 
         with pytest.raises(CompileError) as e:
             qjit(circuit, pipelines=test_pipelines, verbose=True)()

mlir/Makefile

@@ -7,7 +7,7 @@ MK_ABSPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 MK_DIR := $(dir $(MK_ABSPATH))
 DIALECTS_BUILD_DIR?=$(MK_DIR)/build
 LLVM_BUILD_DIR?=$(MK_DIR)/llvm-project/build
-MHLO_BUILD_DIR?=$(MK_DIR)/mlir-hlo/mhlo-build
+MHLO_BUILD_DIR?=$(MK_DIR)/mlir-hlo/bazel-build
 ENZYME_BUILD_DIR?=$(MK_DIR)/Enzyme/build
 RT_BUILD_DIR?=$(MK_DIR)/../runtime/build
 ENABLE_ASAN?=OFF

pyproject.toml

@@ -5,6 +5,7 @@ extend-exclude = '''
   frontend/catalyst/python_bindings
   | mlir/llvm-project
   | mlir/mlir-hlo
+  | mlir/Enzyme
   | build
   | env
 )
@@ -13,7 +14,15 @@ extend-exclude = '''
 [tool.isort]
 profile = "black"
 skip = ["frontend/test/pytest/conftest.py"]
-extend_skip_glob = ["mlir/llvm-project/*", "mlir/mlir-hlo/*", ".git/*", ".vscode/*"]
+extend_skip_glob = [
+  "mlir/llvm-project/*",
+  "mlir/mlir-hlo/*",
+  "mlir/Enzyme/*",
+  "mlir/build/*",
+  "runtime/build/*",
+  ".git/*",
+  ".vscode/*"
+]
 
 [build-system]
 requires = ["setuptools>=62", "wheel", "pybind11>=2.7.0", "numpy>=1.22"]

requirements.txt

@@ -6,7 +6,7 @@ pybind11>=2.8.0
 PyYAML
 
 # formatting/linting
-black[jupyter]
+black
 clang-format==14.*
 clang-tidy==14.*
 pylint
@@ -19,5 +19,7 @@ pytest-xdist
 pytest-cov
 nbmake
 
-# optional rt dependencies
-tensorflow-cpu
+# optional rt/test dependencies
+tensorflow
+amazon-braket-pennylane-plugin>=1.23.0
+pennylane-lightning[kokkos]

runtime/README.rst

@@ -109,7 +109,7 @@ You can use ``ENABLE_LIGHTNING_KOKKOS=ON`` to build the runtime with `lightning.
 
 .. code-block:: console
 
-    ENABLE_LIGHTNING_KOKKOS=ON make runtime
+    make runtime ENABLE_LIGHTNING_KOKKOS=ON
 
 Lightning-Kokkos provides support for other Kokkos backends including OpenMP, HIP and CUDA.
 Please refer to `the installation guideline <https://docs.pennylane.ai/projects/lightning/en/stable/lightning_kokkos/installation.html>`_ for the requirements.
@@ -118,13 +118,13 @@ To build the runtime with Lightning-Kokkos and the ``Kokkos::OpenMP`` backend ex
 
 .. code-block:: console
 
-    ENABLE_LIGHTNING_KOKKOS=ON CMAKE_ARGS="-DKokkos_ENABLE_OPENMP=ON" make runtime
+    make runtime ENABLE_LIGHTNING_KOKKOS=ON CMAKE_ARGS="-DKokkos_ENABLE_OPENMP=ON"
 
 You can also use ``ENABLE_OPENQASM=ON`` to build the runtime with `Amazon-Braket-OpenQasm <https://aws.amazon.com/braket/>`_:
 
 .. code-block:: console
 
-    ENABLE_OPENQASM=ON make runtime
+    make runtime ENABLE_OPENQASM=ON
 
 This device currently offers generators for the `OpenQasm3 <https://openqasm.com/versions/3.0/index.html>`_ specification and
 `Amazon Braket <https://docs.aws.amazon.com/braket/latest/developerguide/braket-openqasm-supported-features.html>`_ assembly extension.
@@ -136,8 +136,8 @@ is required. You can build the runtime with ``BUILD_QIR_STDLIB_FROM_SRC=ON`` aft
 
 .. code-block:: console
 
-  rustup component add llvm-tools-preview
-  BUILD_QIR_STDLIB_FROM_SRC=ON make runtime
+    rustup component add llvm-tools-preview
+    make runtime BUILD_QIR_STDLIB_FROM_SRC=ON
 
 To check the runtime test suite: