From 5512ffb0e2c7bdecb522fa9b98c9276111c4cf61 Mon Sep 17 00:00:00 2001 From: Bryan Van de Ven Date: Tue, 25 Oct 2022 10:44:32 -0700 Subject: [PATCH] Add script to generate conda envs (#367) * Add script to generate conda envs * remove gcc and sysroot pkgs * split out openmpi and compilers options * Adjust consensus match frequency based on field sizes (#402) * Perform consensus match more frequently for bigger free fields * Minor cleanup * add command line args for selection * help wording * Make script executable * Fixes for python 3.8 * Remove old environment files * Unify file naming for "compilers" and "openmpi" * Fix typo * Remove optional ninja dependency * Not just for the core, include cunumeric also * Update build documentation * Fix a file link * Fix formatting * remove typing_extensions dependency * remove jinja dependency * slight vertical whitespace improvemetn * Use custom BooleanFlag action, for the benefit of py3.8 * Update build instructions * Fix intra-document reference * Revise file naming scheme * Update BUILD.md Co-authored-by: Wonchan Lee Co-authored-by: Manolis Papadakis Co-authored-by: Manolis Papadakis --- BUILD.md | 240 ++++++++++++++++++-- README.md | 140 ++---------- conda/environment-test-3.10.yml | 58 ----- conda/environment-test-3.8.yml | 58 ----- conda/environment-test-3.9.yml | 58 ----- scripts/generate-conda-envs.py | 381 ++++++++++++++++++++++++++++++++ 6 files changed, 618 insertions(+), 317 deletions(-) delete mode 100644 conda/environment-test-3.10.yml delete mode 100644 conda/environment-test-3.8.yml delete mode 100644 conda/environment-test-3.9.yml create mode 100755 scripts/generate-conda-envs.py diff --git a/BUILD.md b/BUILD.md index 8bf5f1ac7..5abb0af8d 100644 --- a/BUILD.md +++ b/BUILD.md @@ -15,40 +15,227 @@ limitations under the License. --> -# Overview +# TL;DR -The build system is designed to enable two different modes of use: -1. Simple `install.py` helper script or `pip install` for users -2. Highly customizable incremental builds for developers +1) Check if there are specialized scripts available for your cluster at https://github.com/nv-legate/quickstart. +2) [Install dependencies from conda](#getting-dependencies-through-conda) +3) [Build using install.py](#using-installpy) -We review each of these modes with examples. +# Getting dependencies +## Getting dependencies through conda + +The primary method of retrieving dependencies for Legate Core and downstream +libraries is through [conda](https://conda.io). You will need an installation of +conda to follow the instructions below. + +Please use the `scripts/generate-conda-envs.py` script to create a conda +environment file listing all the packages that are required to build, run and +test Legate Core and all downstream libraries. For example: + +``` +$ ./scripts/generate-conda-envs.py --python 3.10 --ctk 11.7 --os linux --compilers --openmpi +--- generating: environment-test-linux-py310-cuda-11.7-compilers-openmpi.yaml +``` + +Run this script with `-h` to see all available configuration options for the +generated environment file (e.g. all the supported Python versions). See the +[Notable Dependencies](#notable-dependencies) section for more details. + +Once you have this environment file, you can install the required packages by +creating a new conda environment: + +``` +conda env create -n legate -f .yaml +``` + +or by updating an existing environment: + +``` +conda env update -f .yaml +``` + +## Notable dependencies + +### OS (`--os` option) + +Legate has been tested on Linux and MacOS, although only a few flavors of Linux +such as Ubuntu have been thoroughly tested. There is currently no support for +Windows. + +### Python >= 3.8 (`--python` option) + +In terms of Python compatibility, Legate *roughly* follows the timeline outlined +in [NEP 29](https://numpy.org/neps/nep-0029-deprecation_policy.html). + +### C++17 compatible compiler (`--compilers` option) + +For example: g++, clang, or nvc++. When creating an environment using the +`--compilers` flag, an appropriate compiler for the current system will be +pulled from conda. + +If you need/prefer to use the system-provided compilers (typical for HPC +installations), please use a conda environment generated with `--no-compilers`. +Note that this will likely result in a +[conda/system library conflict](#alternative-sources-for-dependencies), +since the system compilers will typically produce executables +that link against the system-provided libraries, which can shadow the +conda-provided equivalents. + +### CUDA >= 10.2 (`--ctk` flag; optional) + +Only necessary if you wish to run with Nvidia GPUs. + +Some CUDA components necessary for building, e.g. the `nvcc` compiler and driver +stubs, are not distributed through conda. These must instead be installed using +[system-level packages](https://developer.nvidia.com/cuda-downloads). + +Independent of the system-level CUDA installation, conda will need to install an +environment-local copy of the CUDA toolkit (which is what the `--ctk` option +controls). To avoid versioning conflicts it is safest to match the version of +CUDA installed system-wide on your machine + +Legate is tested and guaranteed to be compatible with Volta and later GPU +architectures. You can use Legate with Pascal GPUs as well, but there could +be issues due to lack of independent thread scheduling. Please report any such +issues on GitHub. + +### Fortran compiler (optional) + +Only necessary if you wish to build OpenBLAS from source. + +Not included by default in the generated conda environment files; install +`fortran-compiler` from `conda-forge` if you need it. + +### Numactl (optional) + +Required to support CPU and memory binding in the Legate launcher. + +Not available on conda; typically available through the system-level package +manager. + +### MPI (`--openmpi` option) + +Only necessary if you wish to run on multiple nodes. + +Conda distributes a generic build of OpenMPI, but you may need to use a more +specialized build, e.g. the one distributed by +[MOFED](https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/), +or one provided by your HPC vendor. In that case you should use an environment +file generated with `--no-openmpi`. + +Legate requires a build of MPI that supports `MPI_THREAD_MULTIPLE`. + +### Networking libraries (e.g. Infiniband, RoCE, UCX; optional) + +Only necessary if you wish to run on multiple nodes. + +Not available on conda; typically available through MOFED or the system-level +package manager. + +If using UCX, a build configured with `--enable-mt` is required. + +## Alternative sources for dependencies + +If you do not wish to use conda for some (or all) of the dependencies, you can +remove the corresponding entries from the environment file before passing it to +conda. See [the `install.py` section](#using-installpy) for instructions on how +to provide alternative locations for these dependencies to the build process. + +Note that this is likely to result in conflicts between conda-provided and +system-provided libraries. + +Conda distributes its own version of certain common libraries (in particular the +C++ standard library), which are also typically available system-wide. Any +system package you include will typically link to the system version, while +conda packages link to the conda version. Often these two different versions, +although incompatible, carry the same version number (`SONAME`), and are +therefore indistinguishable to the dynamic linker. Then, the first component to +specify a link location for this library will cause it to be loaded from there, +and any subsequent link requests for the same library, even if suggesting a +different link location, will get served using the previously linked version. + +This can cause link failures at runtime, e.g. when a system-level library +happens to be the first to load GLIBC, causing any conda library that comes +after to trip GLIBC's internal version checks, since the conda library expects +to find symbols with more recent version numbers than what is available on the +system-wide GLIBC: + +``` +/lib/x86_64-linux-gnu/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /opt/conda/envs/legate/lib/libarrow.so) +``` + +You can usually work around this issue by putting the conda library directory +first in the dynamic library resolution path: + +``` +LD_LIBRARY_PATH="$CONDA_PREFIX/lib:$LD_LIBRARY_PATH" +``` + +This way you can make sure that the (typically more recent) conda version of any +common library will be preferred over the system-wide one, no matter which +component requests it first. # Building for Users ## Using install.py -For releases <= 22.07, the main method for building Legate was the `install.py` script. -Although the underlying implementation has significantly changed, `install.py` still supports the -same usage and same set of flags. For a full list of flags, users can run: +The Legate Core repository comes with a helper `install.py` script in the +top-level directory, that will build the C++ parts of the library and install +the C++ and Python components under the currently active Python environment. + +To add GPU support, use the `--cuda` flag: + +``` +./install.py --cuda +``` + +You can specify the CUDA toolkit directory and the CUDA architecture you want to +target using the `--with-cuda` and `--arch` flags, e.g.: ``` -$ ./install.py --help +./install.py --cuda --with-cuda /usr/local/cuda/ --arch ampere ``` -## Using Conda +By default the script relies on CMake's auto-detection for these settings. +CMake will first search the currently active Python/conda environment +for dependencies, then any common system-wide installation directories (e.g. +`/usr/lib`). If a dependency cannot be found but is publicly available in source +form (e.g. OpenBLAS), cmake will fetch and build it automatically. You can +override this search by providing an install location for any dependency +explicitly, using a `--with-dep` flag, e.g. `--with-nccl` and +`--with-openblas`. + +For multi-node execution Legate uses [GASNet](https://gasnet.lbl.gov/) which can be +requested using the `--network gasnet1` or `--network gasnetex` flag. By default +GASNet will be automatically downloaded and built, but if you have an existing +installation then you can inform the install script using the `--with-gasnet` flag. +You also need to specify the interconnect network of the target machine using the +`--conduit` flag. + +For example this would be an installation for a +[DGX SuperPOD](https://www.nvidia.com/en-us/data-center/dgx-superpod/): +``` +./install.py --network gasnet1 --conduit ibv --cuda --arch ampere +``` +Alternatively, here is an install line for the +[Piz-Daint](https://www.cscs.ch/computers/dismissed/piz-daint-piz-dora/) supercomputer: +``` +./install.py --network gasnet1 --conduit aries --cuda --arch pascal +``` -Legate can be installed using Conda by pointing to the required channels (`-c`): +To see all available configuration options, run with the `--help` flag: ``` -conda install -c nvidia -c conda-forge -c legate legate-core +./install.py --help ``` ## Using pip -Legate is not yet registered in a standard pip repository. However, users can still use the -pip installer to build and install Legate. After downloading or cloning the legate.core source, -users can run the following in the legate.core folder: +Legate Core is not yet registered in a standard pip repository. However, users +can still use the pip installer to build and install Legate Core. The following +command will trigger a single-node, CPU-only build of Legate Core, then install +it into the currently active Python environment: ``` $ pip install . @@ -58,18 +245,20 @@ or $ python3 -m pip install . ``` -This will install Legate in the standard packages directory for the environment Python. +## Advanced Customization -### Advanced Customization - -If users need to customize details of the underlying CMake build, they can pass -CMake flags through the `SKBUILD_CONFIGURE_OPTIONS` environment variable: +Legate relies on CMake to select its toolchain and build flags. Users can set +the environment variables `CXX` or `CXXFLAGS` prior to building to override the +CMake defaults. Alternatively, CMake values can be overridden through the +`SKBUILD_CONFIGURE_OPTIONS` variable: ``` $ SKBUILD_CONFIGURE_OPTIONS="-D Legion_USE_CUDA:BOOL=ON" \ pip install . ``` + An alternative syntax using `setup.py` with `scikit-build` is + ``` $ python setup.py install -- -DLegion_USE_CUDA:BOOL=ON ``` @@ -86,15 +275,17 @@ in `setup.py` to drive the build and installation. A `pip install` will trigger 3. pip installation of Python files The CMake build can be configured independently of `pip`, allowing incremental C++ builds directly through CMake. -This simplifies rebuilding `libcunumeric.so` either via command-line or via IDE. +This simplifies rebuilding the C++ shared libraries either via command-line or via IDE. After building the C++ libraries, the `pip install` can be done in "editable" mode using the `-e` flag. This configures the Python site packages to import the Python source tree directly. The Python source can then be edited and used directly for testing without requiring another `pip install`. ## Example -There are several examples in the `scripts` folder. We walk through the steps in the `build-separately-no-install.sh` here. -First, the CMake build needs to be configured, e.g.: +There are several examples in the `scripts` folder. We walk through the steps in +`build-separately-no-install.sh` here. + +First, the CMake build needs to be configured: ``` $ cmake -S . -B build -GNinja -D Legion_USE_CUDA=ON @@ -118,6 +309,7 @@ $ SKBUILD_BUILD_OPTIONS="-D FIND_LEGATE_CORE_CPP=ON -D legate_core_ROOT=$(pwd)/b The Python source tree and CMake build tree are now available with the environment Python for running Legate programs. The diagram below illustrates the -complete workflow for building both Legate core and a downstream package [cuNumeric]() +complete workflow for building both Legate core and a downstream package, +[cuNumeric](https://github.com/nv-legate/cunumeric) drawing diff --git a/README.md b/README.md index ff1142695..fe0d5b5e4 100644 --- a/README.md +++ b/README.md @@ -50,15 +50,23 @@ Pull requests are welcomed. If you have questions, please contact us at legate(at)nvidia.com. -1. [Why Legate?](#why-legate) -1. [What is the Legate Core?](#what-is-the-legate-core) -1. [How Does Legate Work?](#how-does-legate-work) -1. [How Do I Install Legate?](#how-do-i-install-legate) -1. [How Do I Use Legate?](#how-do-i-use-legate) -1. [Other FAQs](#other-faqs) -1. [Contributing](#contributing) -1. [Documentation](#documentation) -1. [Next Steps](#next-steps) +- [Legate](#legate) + - [Why Legate?](#why-legate) + - [What is the Legate Core?](#what-is-the-legate-core) + - [How Does Legate Work?](#how-does-legate-work) + - [How Do I Install Legate?](#how-do-i-install-legate) + - [How Do I Use Legate?](#how-do-i-use-legate) + - [Distributed Launch](#distributed-launch) + - [Debugging and Profiling](#debugging-and-profiling) + - [Running Legate programs with Jupyter Notebook](#running-legate-programs-with-jupyter-notebook) + - [Installation of the Legate IPython Kernel](#installation-of-the-legate-ipython-kernel) + - [Running with Jupyter Notebook](#running-with-jupyter-notebook) + - [Configuring the Jupyter Notebook](#configuring-the-jupyter-notebook) + - [Magic Command](#magic-command) + - [Other FAQs](#other-faqs) + - [Contributing](#contributing) + - [Documentation](#documentation) + - [Next Steps](#next-steps) ## Why Legate? @@ -215,120 +223,14 @@ Legate Core is available [on conda](https://anaconda.org/legate/legate-core): conda install -c nvidia -c conda-forge -c legate legate-core ``` +The conda package is compatible with CUDA >= 11.4 (CUDA driver version >= r470), +and Volta or later GPU architectures. + Docker image build scripts, as well as specialized install scripts for supported clusters are available on the [quickstart](https://github.com/nv-legate/quickstart) repo. -Read on for general instructions on building Legate Core from source. - -### Dependencies - -Legate has been tested on Linux and MacOS, although only a few flavors of Linux -such as Ubuntu have been thoroughly tested. There is currently no support for -Windows. - -Legate Core requires the following: - - - Python >= 3.8 - - [CUDA](https://developer.nvidia.com/cuda-downloads) >= 10.2 - - GNU Make - - C++17 compatible compiler (g++, clang, or nvc++) - - numactl (optional, to support CPU and memory binding) - - the Python packages listed in any one of the conda environment files: - - `conda/environment-test-3.8.yml` - - `conda/environment-test-3.9.yml` - - `conda/environment-test-3.10.yml` - -You can install the required Python packages by creating a new conda environment: - -``` -conda env create -n legate -f conda/environment-test-3.10.yml -``` - -or by updating an existing environment: - -``` -conda env update -f conda/environment-test-3.10.yml -``` - -Note that conda will need to install an environment-local copy of the CUDA -toolkit, and by default it will choose the latest available version. To avoid -versioning conflicts, however, it is safer to match the version of CUDA -installed system-wide on your machine. Therefore, we suggest that you add this -as an explicit dependency at the bottom of the conda environment file. For -example, if your system-wide CUDA installation is at version 10.2, add: - -``` - - cudatoolkit=10.2 -``` - -### Installation - -The Legate Core library comes with both a standard `setup.py` script and a -custom `install.py` script in the top-level directory of the repository that -will build and install the Legate Core library. Users can use either script -to install Legate as they will produce the same effect. Users can do a simple -pip installation of a single-node, CPU-only Legate configuration by navigating -to the Legate source directory and running: -``` -pip install . -``` -or -``` -python3 -m pip install . -``` - -This will install Legate into the standard packages of the Python environment. - -To add GPU support or do more complicated customization, Legate provides a -helper `install.py` script. For GPU support, simply use the `--cuda` flag: - -``` -./install.py --cuda -``` - -The first time you request GPU support you may need to use the `--with-cuda` flag to -specify the location of your CUDA installation and the `--with-nccl` flag to specify -the path to your NCCL installation, if these cannot be automatically located by the build system. -You can also specify the name of the CUDA architecture you want to target with the `--arch` -flag. By default the script relies on CMake's auto-detection. -``` -./install.py --cuda --with-cuda /usr/local/cuda/ --with-nccl "$CONDA_PREFIX" --arch ampere -``` -For multi-node support Legate uses [GASNet](https://gasnet.lbl.gov/) which can be -requested using the `--network gasnet1` or `--network gasnetex` flag. By default -GASNet will be automatically downloaded and built, but if you have an existing -installation then you can inform the install script using the `--with-gasnet` flag. -You also need to specify the interconnect network of the target machine using the -`--conduit` flag. - -For example this would be an installation for a -[DGX SuperPOD](https://www.nvidia.com/en-us/data-center/dgx-superpod/): -``` -./install.py --network gasnet1 --conduit ibv --cuda --arch ampere -``` -Alternatively here is an install line for the -[Piz-Daint](https://www.cscs.ch/computers/dismissed/piz-daint-piz-dora/) supercomputer: -``` -./install.py --network gasnet1 --conduit aries --cuda --arch pascal -``` -To see all the options available for installing Legate, run with the `--help` flag: -``` -./install.py --help -``` - -### Toolchain Selection - -Legate relies on CMake to select its toolchain and build flags. -Users can set the environment variables `CXX` or `CXXFLAGS` -prior to building to override the CMake defaults. Alternatively, CMake values -can be overriden through the `SKBUILD_CONFIGURE_OPTIONS` variable, -which is discussed in more detail in the [developer build instructions](BUILD.md). - -### Developer Workflow - -Details on doing incremental CMake builds and editable pip installations can be -found in the [developer build instructions](BUILD.md). +See [BUILD.md]() for instructions on building Legate Core from source. ## How Do I Use Legate? diff --git a/conda/environment-test-3.10.yml b/conda/environment-test-3.10.yml deleted file mode 100644 index 736e4c5e0..000000000 --- a/conda/environment-test-3.10.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: legate-core-test -channels: - - conda-forge -dependencies: - - python=3.10 - - # build - - git - - nccl - - make - - zlib - - cmake>=3.24 - - ninja - - openmpi - - c-compiler - - cxx-compiler - - gcc_linux-64 # [linux64] - - sysroot_linux-64==2.17 # [linux64] - - setuptools>=60 - - scikit-build>=0.13.1 - - # runtime - - cffi - - numpy>=1.22 - - opt_einsum - - pyarrow>=5 - - scipy - - typing_extensions - - llvm-openmp - - # tests - - clang>=8 - - clang-tools>=8 - - colorama - - coverage - - mock - - mypy>=0.961 - - pre-commit - - pynvml - - pytest - - pytest-cov - - pytest-lazy-fixture - - types-docutils - - # pip dependencies - - pip - - pip: - # docs - - jinja2 - - pydata-sphinx-theme - - recommonmark - - markdown<3.4.0 - - sphinx>=4.4.0 - - sphinx-copybutton - - sphinx-markdown-tables - - # examples - - tifffile diff --git a/conda/environment-test-3.8.yml b/conda/environment-test-3.8.yml deleted file mode 100644 index 9f58e9b5d..000000000 --- a/conda/environment-test-3.8.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: legate-core-test -channels: - - conda-forge -dependencies: - - python=3.8 - - # build - - git - - nccl - - make - - zlib - - cmake>=3.24 - - ninja - - openmpi - - c-compiler - - cxx-compiler - - gcc_linux-64 # [linux64] - - sysroot_linux-64==2.17 # [linux64] - - setuptools>=60 - - scikit-build>=0.13.1 - - # runtime - - cffi - - numpy>=1.22 - - opt_einsum - - pyarrow>=5 - - scipy - - typing_extensions - - llvm-openmp - - # tests - - clang>=8 - - clang-tools>=8 - - colorama - - coverage - - mock - - mypy>=0.961 - - pre-commit - - pynvml - - pytest - - pytest-cov - - pytest-lazy-fixture - - types-docutils - - # pip dependencies - - pip - - pip: - # docs - - jinja2 - - pydata-sphinx-theme - - recommonmark - - markdown<3.4.0 - - sphinx>=4.4.0 - - sphinx-copybutton - - sphinx-markdown-tables - - # examples - - tifffile diff --git a/conda/environment-test-3.9.yml b/conda/environment-test-3.9.yml deleted file mode 100644 index 9d4eea27d..000000000 --- a/conda/environment-test-3.9.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: legate-core-test -channels: - - conda-forge -dependencies: - - python=3.9 - - # build - - git - - nccl - - make - - zlib - - cmake>=3.24 - - ninja - - openmpi - - c-compiler - - cxx-compiler - - gcc_linux-64 # [linux64] - - sysroot_linux-64==2.17 # [linux64] - - setuptools>=60 - - scikit-build>=0.13.1 - - # runtime - - cffi - - numpy>=1.22 - - opt_einsum - - pyarrow>=5 - - scipy - - typing_extensions - - llvm-openmp - - # tests - - clang>=8 - - clang-tools>=8 - - colorama - - coverage - - mock - - mypy>=0.961 - - pre-commit - - pynvml - - pytest - - pytest-cov - - pytest-lazy-fixture - - types-docutils - - # pip dependencies - - pip - - pip: - # docs - - jinja2 - - pydata-sphinx-theme - - recommonmark - - markdown<3.4.0 - - sphinx>=4.4.0 - - sphinx-copybutton - - sphinx-markdown-tables - - # examples - - tifffile diff --git a/scripts/generate-conda-envs.py b/scripts/generate-conda-envs.py new file mode 100755 index 000000000..a5cd426ee --- /dev/null +++ b/scripts/generate-conda-envs.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved. +# +# NVIDIA CORPORATION and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto. Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA CORPORATION is strictly prohibited. +# +# See the LICENSE file for details. +# +from __future__ import annotations + +from argparse import Action, ArgumentParser +from dataclasses import dataclass +from textwrap import indent +from typing import Literal, Protocol, Tuple + +# --- Types ------------------------------------------------------------------- + +Req = str +Reqs = Tuple[Req, ...] +OSType = Literal["linux", "darwin"] + + +class SectionConfig(Protocol): + header: str + + @property + def conda(self) -> Reqs: + return () + + @property + def pip(self) -> Reqs: + return () + + def __str__(self) -> str: + return self.header + + def format(self, kind: str) -> str: + return SECTION_TEMPLATE.format( + header=self.header, + reqs="- " + + "\n- ".join(self.conda if kind == "conda" else self.pip), + ) + + +@dataclass(frozen=True) +class CUDAConfig(SectionConfig): + ctk_version: str | None + + header = "cuda" + + @property + def conda(self) -> Reqs: + if self.ctk_version is None: + return () + + return ( + f"cudatoolkit={self.ctk_version}", # runtime + "cutensor>=1.3.3", # runtime + "nccl", # runtime + "pynvml", # tests + ) + + def __str__(self) -> str: + if self.ctk_version == "none": + return "" + + return f"-cuda{self.ctk_version}" + + +@dataclass(frozen=True) +class BuildConfig(SectionConfig): + compilers: bool = True + openmpi: bool = True + + header = "build" + + @property + def conda(self) -> Reqs: + pkgs = ( + "cmake>=3.24", + "git", + "make", + "scikit-build>=0.13.1", + "setuptools>=60", + "zlib", + ) + if self.compilers: + pkgs += ("c-compiler", "cxx-compiler") + if self.openmpi: + pkgs += ("openmpi",) + return sorted(pkgs) + + def __str__(self) -> str: + val = "-compilers" if self.compilers else "" + val += "-openmpi" if self.openmpi else "" + return val + + +@dataclass(frozen=True) +class RuntimeConfig(SectionConfig): + header = "runtime" + + @property + def conda(self) -> Reqs: + return ( + "cffi", + "llvm-openmp", + "numpy>=1.22", + "openblas=*=*openmp*", + "opt_einsum", + "pyarrow>=5", + "scipy", + "typing_extensions", + ) + + +@dataclass(frozen=True) +class TestsConfig(SectionConfig): + header = "tests" + + @property + def conda(self) -> Reqs: + return ( + "clang-tools>=8", + "clang>=8", + "colorama", + "coverage", + "mock", + "mypy>=0.961", + "pre-commit", + "pytest-cov", + "pytest-lazy-fixture", + "pytest-mock", + "pytest", + "types-docutils", + ) + + @property + def pip(self) -> Reqs: + return ("tifffile",) + + +@dataclass(frozen=True) +class DocsConfig(SectionConfig): + header = "docs" + + @property + def pip(self) -> Reqs: + return ( + "jinja2", + "markdown<3.4.0", + "pydata-sphinx-theme", + "recommonmark", + "sphinx-copybutton", + "sphinx-markdown-tables", + "sphinx>=4.4.0", + ) + + +@dataclass(frozen=True) +class EnvConfig: + use: str + python: str + os: OSType + ctk: str | None + compilers: bool + openmpi: bool + + @property + def sections(self) -> Tuple[SectionConfig, ...]: + return ( + self.cuda, + self.build, + self.runtime, + self.tests, + self.docs, + ) + + @property + def cuda(self) -> CUDAConfig: + return CUDAConfig(self.ctk) + + @property + def build(self) -> BuildConfig: + return BuildConfig(self.compilers, self.openmpi) + + @property + def runtime(self) -> RuntimeConfig: + return RuntimeConfig() + + @property + def tests(self) -> TestsConfig: + return TestsConfig() + + @property + def docs(self) -> DocsConfig: + return DocsConfig() + + @property + def filename(self) -> str: + return f"environment-{self.use}-{self.os}-py{self.python}{self.cuda}{self.build}.yaml" # noqa + + +# --- Setup ------------------------------------------------------------------- + +PYTHON_VERSIONS = ("3.8", "3.9", "3.10") + +CTK_VERSIONS = ( + "none", + "10.2", + "11.0", + "11.1", + "11.2", + "11.3", + "11.4", + "11.5", + "11.6", + "11.7", +) + +OS_NAMES: Tuple[OSType, ...] = ("linux", "osx") + + +ENV_TEMPLATE = """\ +name: legate-{use} +channels: + - conda-forge +dependencies: + + - python={python} + +{conda_sections}{pip} +""" + +SECTION_TEMPLATE = """\ +# {header} +{reqs} + +""" + +PIP_TEMPLATE = """\ + - pip + - pip: +{pip_sections} +""" + +ALL_CONFIGS = [ + EnvConfig("test", python, "linux", ctk, compilers, openmpi) + for python in PYTHON_VERSIONS + for ctk in CTK_VERSIONS + for compilers in (True, False) + for openmpi in (True, False) +] + [ + EnvConfig("test", python, "darwin", "none", compilers, openmpi) + for python in PYTHON_VERSIONS + for compilers in (True, False) + for openmpi in (True, False) +] + +# --- Code -------------------------------------------------------------------- + + +class BooleanFlag(Action): + def __init__( + self, + option_strings, + dest, + default, + required=False, + help="", + metavar=None, + ): + assert all(not opt.startswith("--no") for opt in option_strings) + + def flatten(list): + return [item for sublist in list for item in sublist] + + option_strings = flatten( + [ + [opt, "--no-" + opt[2:], "--no" + opt[2:]] + if opt.startswith("--") + else [opt] + for opt in option_strings + ] + ) + super().__init__( + option_strings, + dest, + nargs=0, + const=None, + default=default, + type=bool, + choices=None, + required=required, + help=help, + metavar=metavar, + ) + + def __call__(self, parser, namespace, values, option_string): + setattr(namespace, self.dest, not option_string.startswith("--no")) + + +if __name__ == "__main__": + + import sys + + parser = ArgumentParser() + parser.add_argument( + "--python", + choices=PYTHON_VERSIONS, + default=None, + help="Python version to generate for, (default: all python versions)", + ) + parser.add_argument( + "--ctk", + choices=CTK_VERSIONS, + default=None, + dest="ctk_version", + help="CTK version to generate for (default: all CTK versions)", + ) + parser.add_argument( + "--os", + choices=OS_NAMES, + default=None, + help="OS to generate for (default: all OSes)", + ) + parser.add_argument( + "--compilers", + action=BooleanFlag, + dest="compilers", + default=None, + help="Whether to include conda compilers or not (default: both)", + ) + parser.add_argument( + "--openmpi", + action=BooleanFlag, + dest="openmpi", + default=None, + help="Whether to include openmpi or not (default: both)", + ) + + args = parser.parse_args(sys.argv[1:]) + + configs = ALL_CONFIGS + + if args.python is not None: + configs = (x for x in configs if x.python == args.python) + if args.ctk_version is not None: + configs = ( + x for x in configs if x.cuda.ctk_version == args.ctk_version + ) + if args.compilers is not None: + configs = (x for x in configs if x.build.compilers == args.compilers) + if args.os is not None: + configs = (x for x in configs if x.os == args.os) + if args.openmpi is not None: + configs = (x for x in configs if x.build.openmpi == args.openmpi) + + for config in configs: + conda_sections = indent( + "".join(s.format("conda") for s in config.sections if s.conda), + " ", + ) + + pip_sections = indent( + "".join(s.format("pip") for s in config.sections if s.pip), " " + ) + + print(f"--- generating: {config.filename}") + out = ENV_TEMPLATE.format( + use=config.use, + python=config.python, + conda_sections=conda_sections, + pip=PIP_TEMPLATE.format(pip_sections=pip_sections), + ) + with open(f"{config.filename}", "w") as f: + f.write(out)