From a7f34bf3f01cccfcf4178400de47827f601e5fd7 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:03:44 +0100 Subject: [PATCH 01/10] add makefile and update readme --- Makefile | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 28 ++++++++++++++++++---------- 2 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..1c73a396b4 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +MAKEFLAGS += --no-builtin-rules +MAKEFLAGS += --no-builtin-variables + +xdsl_sources = $(shell find xdsl -type f) +test_sources = $(shell find tests -type f) +filecheck_sources = $(shell find tests -iname '*.mlir' -type f) + + +.PHONY: clean filecheck pytest tests rerun-notebooks precommit-install precommit black pyright + +# remove all caches and the venv +clean: + rm -rf venv .pytest_cache *.egg-info .coverage.* + +# run filecheck tests +filecheck: + lit -vv tests/filecheck + +# run pytest tests +pytest: + pytest tests -W error -vv + +# run all tests +tests: pytest filecheck + echo test + +# re-generate the output from all jupyter notebooks in the docs directory +rerun-notebooks: + jupyter nbconvert --ClearMetadataPreprocessor.enabled=True --inplace --to notebook --execute docs/*.ipynb + +# set up all precommit hooks +precommit-install: + pre-commit install + +# run all precommit hooks and apply them +precommit: + pre-commit run --all + +# run pyright on all files in the current git commit +pyright: + pyright $(shell git diff --staged --name-only) + +# run black on all files currently staged +black: + black $(shell git diff --staged --name-only) + +# set up the venv with all dependencies for development +venv: requirements-optional.txt requirements.txt + python3 -m venv venv + source venv/bin/activate + pip install -r requirements-optional.txt requirements.txt + pip install -e ".[extras]" diff --git a/README.md b/README.md index 44fa09729f..b8c3e77105 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ for users interested in that use case. - [A DSL for defining new IRs](https://xdsl.dev/xdsl/retro/notebooks/?path=irdl.ipynb) - [Connecting xDSL with MLIR](docs/mlir_interoperation.md) +We provide a Makefile containing a lot of common tasks, which might provide +an overview of common actions. + ## xDSL Developer Setup To contribute to the development of xDSL follow the subsequent steps. @@ -67,7 +70,8 @@ To contribute to the development of xDSL follow the subsequent steps. ```bash git clone https://github.com/xdslproject/xdsl.git cd xdsl -pip install --editable ".[extras]" +# set up the venv and install everything +make venv ``` ### Testing @@ -81,9 +85,12 @@ pytest # Executes filecheck tests lit tests/filecheck + +# run all tests using makefile +make tests ``` -### Formatting +### Formatting and Typechecking All python code used in xDSL uses [black](https://github.com/psf/black) to format the code in a uniform manner. @@ -92,17 +99,18 @@ To automate the formatting, we use pre-commit hooks from the [pre-commit](https://pypi.org/project/pre-commit/) package. ```bash -# Install pre-commit version in the optional requirement files -pip install --requirement requirements-optional.txt - # Install the pre-commit on your `.git` folder -# Be sure to run this in your virtual environment -pre-commit install - -# Optionally test the hook by running it on all files -pre-commit run --all-files +make precommit-install +# to run the hooks: +make precommit +# alternatively, running black on all staged files: +make black # or simply black $(git diff --staged --name-only) ``` +Furthermore, all python code must run through [pyright](https://github.com/microsoft/pyright) +without errors. Pyright can be run on all staged files through the +makefile using `make pyright`. + ### Discussion You can also join the discussion at our [Zulip chat room](https://xdsl.zulipchat.com), kindly supported by community hosting from [Zulip](https://zulip.com/). From 4ed3a54d138ad9bea200d44fa7b3c63c2a06ec72 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:06:46 +0100 Subject: [PATCH 02/10] remove old cruft from Makefile --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index 1c73a396b4..18851bd638 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,6 @@ MAKEFLAGS += --no-builtin-rules MAKEFLAGS += --no-builtin-variables -xdsl_sources = $(shell find xdsl -type f) -test_sources = $(shell find tests -type f) -filecheck_sources = $(shell find tests -iname '*.mlir' -type f) - .PHONY: clean filecheck pytest tests rerun-notebooks precommit-install precommit black pyright From 367e4d8b9759711ac750049168e0ea1d5ce33bc6 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:14:10 +0100 Subject: [PATCH 03/10] fix venv setup in makefile, add pyright to tests targets --- Makefile | 7 +++++-- README.md | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 18851bd638..f54416495e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,10 @@ MAKEFLAGS += --no-builtin-rules MAKEFLAGS += --no-builtin-variables +# make tasks runn all commands in a single shell +.ONESHELL: +# these targets don't produce files: .PHONY: clean filecheck pytest tests rerun-notebooks precommit-install precommit black pyright # remove all caches and the venv @@ -17,7 +20,7 @@ pytest: pytest tests -W error -vv # run all tests -tests: pytest filecheck +tests: pytest filecheck pyright echo test # re-generate the output from all jupyter notebooks in the docs directory @@ -44,5 +47,5 @@ black: venv: requirements-optional.txt requirements.txt python3 -m venv venv source venv/bin/activate - pip install -r requirements-optional.txt requirements.txt + pip install -r requirements-optional.txt -r requirements.txt pip install -e ".[extras]" diff --git a/README.md b/README.md index b8c3e77105..ec2feb1d0d 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ git clone https://github.com/xdslproject/xdsl.git cd xdsl # set up the venv and install everything make venv +# activate the venv +source venv/bin/activate ``` ### Testing From 811615418bb0a8da601fccae21be501cdb7ed3b8 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:15:53 +0100 Subject: [PATCH 04/10] Update Makefile to include toy tests Co-authored-by: Sasha Lopoukhine --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index f54416495e..831f3d58e7 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ clean: # run filecheck tests filecheck: lit -vv tests/filecheck + lit -vv docs/Toy/examples # run pytest tests pytest: From 78514ce6e979d3fa323acb4cf9da25a16ff4a71f Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:18:33 +0100 Subject: [PATCH 05/10] allow overriding venv dir name --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 831f3d58e7..4214bde7a6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,10 @@ MAKEFLAGS += --no-builtin-rules MAKEFLAGS += --no-builtin-variables -# make tasks runn all commands in a single shell +# allow overriding the name of the venv directory +VENV_DIR ?= venv + +# make tasks run all commands in a single shell .ONESHELL: # these targets don't produce files: @@ -9,7 +12,7 @@ MAKEFLAGS += --no-builtin-variables # remove all caches and the venv clean: - rm -rf venv .pytest_cache *.egg-info .coverage.* + rm -rf ${VENV_DIR} .pytest_cache *.egg-info .coverage.* # run filecheck tests filecheck: @@ -46,7 +49,7 @@ black: # set up the venv with all dependencies for development venv: requirements-optional.txt requirements.txt - python3 -m venv venv - source venv/bin/activate + python3 -m venv ${VENV_DIR} + source ${VENV_DIR}/bin/activate pip install -r requirements-optional.txt -r requirements.txt pip install -e ".[extras]" From 761008da475c27929bd4cb4d76069cf0b3097cd9 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:30:27 +0100 Subject: [PATCH 06/10] add notebooks to pytests --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 4214bde7a6..6bae91fb5d 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ filecheck: # run pytest tests pytest: pytest tests -W error -vv + pytest -W error --nbval -vv docs --ignore=docs/mlir_interoperation.ipynb # run all tests tests: pytest filecheck pyright From b1d6d608bc8ac3d3d3aa587aa8efa7937ffa6cd8 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:32:44 +0100 Subject: [PATCH 07/10] move notebook tests to separate target --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6bae91fb5d..9d6fdf98cd 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ VENV_DIR ?= venv .ONESHELL: # these targets don't produce files: -.PHONY: clean filecheck pytest tests rerun-notebooks precommit-install precommit black pyright +.PHONY: clean filecheck pytest pytest-nb tests rerun-notebooks precommit-install precommit black pyright # remove all caches and the venv clean: @@ -22,10 +22,13 @@ filecheck: # run pytest tests pytest: pytest tests -W error -vv + +# run pytest on notebooks +pytest-nb: pytest -W error --nbval -vv docs --ignore=docs/mlir_interoperation.ipynb # run all tests -tests: pytest filecheck pyright +tests: pytest pytest-nb filecheck pyright echo test # re-generate the output from all jupyter notebooks in the docs directory From 6f66f969ffd4664968a3c768ed8aa39c6e48c165 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:37:06 +0100 Subject: [PATCH 08/10] make black target default to all of xdsl --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9d6fdf98cd..99dbb653cf 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,9 @@ pyright: # run black on all files currently staged black: - black $(shell git diff --staged --name-only) + touched="$(shell git diff --staged --name-only)" + black $${touched:-xdsl} + # set up the venv with all dependencies for development venv: requirements-optional.txt requirements.txt From b37dc7dbac6169342490fbe2a34f00c658e55f12 Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:37:54 +0100 Subject: [PATCH 09/10] Makefile: better naming --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 99dbb653cf..9411f94419 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,9 @@ pyright: # run black on all files currently staged black: - touched="$(shell git diff --staged --name-only)" - black $${touched:-xdsl} + staged_files="$(shell git diff --staged --name-only)" + # run black on all of xdsl if no staged files exist + black $${staged_files:-xdsl} # set up the venv with all dependencies for development From e8feeb31688c54aeb7fb00c0fe5b89dca8d9033a Mon Sep 17 00:00:00 2001 From: Anton Lydike Date: Wed, 21 Jun 2023 15:39:04 +0100 Subject: [PATCH 10/10] Makefile: formatting --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 9411f94419..fa5694710a 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,6 @@ black: # run black on all of xdsl if no staged files exist black $${staged_files:-xdsl} - # set up the venv with all dependencies for development venv: requirements-optional.txt requirements.txt python3 -m venv ${VENV_DIR}