From 15785bd0a5fb1256a28079194e90fb371e845e54 Mon Sep 17 00:00:00 2001 From: Emilien Bauer Date: Thu, 12 Jan 2023 15:18:57 +0000 Subject: [PATCH 1/5] Build a xDSL-embedding Pyodide distribution. --- .github/workflows/jupyterlite.yml | 83 ++++++++++++++++--------------- docs/irdl.ipynb | 2 - docs/tutorial.ipynb | 2 - 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/.github/workflows/jupyterlite.yml b/.github/workflows/jupyterlite.yml index ee5447b971..7bced10ce7 100644 --- a/.github/workflows/jupyterlite.yml +++ b/.github/workflows/jupyterlite.yml @@ -11,6 +11,8 @@ jobs: steps: - name: Checkout xDSL uses: actions/checkout@v3 + with: + path: xdsl - name: Setup Python uses: actions/setup-python@v4 @@ -18,61 +20,60 @@ jobs: python-version: '3.10' - name: Install dependencies - run: python -m pip install jupyterlite[all] libarchive-c jupyter_server jupyterlab_server build pkginfo pyodide-build - - - name: Setup EmSDK - # Doing this in a hacky script because the custom Action is requiring an explicit EmSDK version - # Pyodide is requiring a specific version too and it is changing regularly. This install the actual - # version required by the latest pyodide. - - # Double initialization of PYODIDE_EMSCRIPTEN_VERSION: see https://github.com/pyodide/pyodide/issues/3430 run: | - git clone https://github.com/emscripten-core/emsdk.git - cd emsdk - PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) - PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) - ./emsdk install ${PYODIDE_EMSCRIPTEN_VERSION} - ./emsdk activate ${PYODIDE_EMSCRIPTEN_VERSION} - - - name: Build WASM wheels for frozenlist, coverage, and xDSL + python -m pip install jupyterlite[all] libarchive-c build pyodide-build + + - name: Build xDSL source distribution run: | - mkdir pypi - source emsdk/emsdk_env.sh - for i in frozenlist coverage - do - - python -m pip download --no-binary :all: "$(grep $i requirements.txt)" - tar -xf $i-*.tar.gz - rm $i_*.tar.gz - cd $i-* - pyodide build - cd .. - cp $(find -path "./$i-*/dist/$i-*_wasm32.whl") pypi + cd xdsl + python setup.py sdist - done + - name: Restore cached Pyodide tree + id: cache-pyodide + uses: actions/cache@v3 + with: + path: pyodide + key: pyodide + + - name: Clone pyodide if not cached + if: steps.cache-pyodide.outputs.cache-hit != 'true' + run: git clone https://github.com/pyodide/pyodide.git - pyodide build - cp $(find -path "./dist/xdsl-*.whl") pypi + - name: Build custom Pyodide distribution + run: | + XDSL_SOURCE_DIST=$(ls xdsl/dist) + XDSL_SOURCE_URL=file://$(pwd)/xdsl/dist/$XDSL_SOURCE_DIST + XDSL_SHA256=($(sha256sum xdsl/dist/$XDSL_SOURCE_DIST)) + + cd pyodide + git fetch --all + git checkout 0.22.0a3 + python -m pip install -r requirements.txt + sudo apt update && sudo apt install f2c + + rm packages/xdsl packages/frozenlist -rf + pyodide skeleton pypi xdsl + pyodide skeleton pypi frozenlist + + python -c 'import yaml; f = open("packages/xdsl/meta.yaml"); y = yaml.safe_load(f); f.close(); y["requirements"] = {"run" : ["frozenlist"]}; y["source"] = {"url" : "'$XDSL_SOURCE_URL'", "sha256" : "'$XDSL_SHA256'"}; f = open("packages/xdsl/meta.yaml", "w"); yaml.dump(y, f)' + + PYODIDE_PACKAGES="frozenlist,coverage,xdsl" make - name: Build the JupyterLite site run: | mkdir content - cp README.md content - cp docs content -r - for f in $(find -path ./content/*.ipynb) - do - - echo appending to $f - sed -i 's/# xDSL should be available in the environment\./# xDSL should be available in the environment.\\n%pip install xdsl/g' $f + cp xdsl/docs/* content -r - done + rm -rf pyodide/pyodide + mkdir pyodide/pyodide + mv pyodide/dist pyodide/pyodide/pyodide - python -m jupyter lite build --contents content --output-dir jupyterlite + python -m jupyter lite build --contents content --pyodide pyodide/pyodide - name: Upload artifact uses: actions/upload-pages-artifact@v1 with: - path: ./jupyterlite + path: ./_output deploy: needs: build diff --git a/docs/irdl.ipynb b/docs/irdl.ipynb index 56abc5ca29..7bfd1b2f02 100644 --- a/docs/irdl.ipynb +++ b/docs/irdl.ipynb @@ -26,8 +26,6 @@ "metadata": {}, "outputs": [], "source": [ - "# xDSL should be available in the environment.\n", - "\n", "from xdsl.ir import MLContext\n", "from xdsl.dialects.arith import Arith\n", "from xdsl.dialects.builtin import Builtin\n", diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb index 6eda62fa7f..5a265e14c1 100644 --- a/docs/tutorial.ipynb +++ b/docs/tutorial.ipynb @@ -25,8 +25,6 @@ "metadata": {}, "outputs": [], "source": [ - "# xDSL should be available in the environment.\n", - "\n", "from xdsl.ir import MLContext\n", "from xdsl.dialects.arith import Arith\n", "from xdsl.dialects.func import Func\n", From c9c256ff40cdeeed96d9de59a99c237911c3d99a Mon Sep 17 00:00:00 2001 From: Emilien Bauer Date: Thu, 12 Jan 2023 15:55:01 +0000 Subject: [PATCH 2/5] Update README.md for new JupyterLite location --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e5b0026ce0..93b212e097 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ The following tutorials will present xDSL basic concepts, how to use its irdl dialect to define new dialects in a user-friendly way, and how to work with both xDSL and MLIR. -- [A simple introduction](https://xdsl.dev/xdsl/retro/notebooks/?path=docs/tutorial.ipynb) -- [A DSL for defining new IRs](https://xdsl.dev/xdsl/retro/notebooks/?path=docs/irdl.ipynb) +- [A simple introduction](https://xdsl.dev/xdsl/retro/notebooks/?path=tutorial.ipynb) +- [A DSL for defining new IRs](https://xdsl.dev/xdsl/retro/notebooks/?path=irdl.ipynb) - [Connecting xDSL with MLIR](docs/mlir_interoperation.md) ## xDSL Developer Setup From e86c651234b3f86ffbccd196e5b61cd11fabdf07 Mon Sep 17 00:00:00 2001 From: Emilien Bauer Date: Thu, 12 Jan 2023 17:03:40 +0000 Subject: [PATCH 3/5] Move the YAML updating to a Python script. --- .github/workflows/jupyterlite.yml | 10 +++-- .../workflows/update_xdsl_pyodide_build.py | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100755 .github/workflows/update_xdsl_pyodide_build.py diff --git a/.github/workflows/jupyterlite.yml b/.github/workflows/jupyterlite.yml index 7bced10ce7..9fa2ffc043 100644 --- a/.github/workflows/jupyterlite.yml +++ b/.github/workflows/jupyterlite.yml @@ -28,6 +28,9 @@ jobs: cd xdsl python setup.py sdist + # Pyodide is cached, so cloned only if not present in the cache, otherwise + # just checked out to whatever desired version and partially rebuilt. + - name: Restore cached Pyodide tree id: cache-pyodide uses: actions/cache@v3 @@ -39,11 +42,10 @@ jobs: if: steps.cache-pyodide.outputs.cache-hit != 'true' run: git clone https://github.com/pyodide/pyodide.git + # Clean the xDSL and FrozenList package folders, generate their skeletons + # and do the necessary updates before building. - name: Build custom Pyodide distribution run: | - XDSL_SOURCE_DIST=$(ls xdsl/dist) - XDSL_SOURCE_URL=file://$(pwd)/xdsl/dist/$XDSL_SOURCE_DIST - XDSL_SHA256=($(sha256sum xdsl/dist/$XDSL_SOURCE_DIST)) cd pyodide git fetch --all @@ -55,7 +57,7 @@ jobs: pyodide skeleton pypi xdsl pyodide skeleton pypi frozenlist - python -c 'import yaml; f = open("packages/xdsl/meta.yaml"); y = yaml.safe_load(f); f.close(); y["requirements"] = {"run" : ["frozenlist"]}; y["source"] = {"url" : "'$XDSL_SOURCE_URL'", "sha256" : "'$XDSL_SHA256'"}; f = open("packages/xdsl/meta.yaml", "w"); yaml.dump(y, f)' + ../xdsl/.github/workflows/update_xdsl_pyodide_build.py packages/xdsl/meta.yaml ../xdsl PYODIDE_PACKAGES="frozenlist,coverage,xdsl" make diff --git a/.github/workflows/update_xdsl_pyodide_build.py b/.github/workflows/update_xdsl_pyodide_build.py new file mode 100755 index 0000000000..27e3b7084f --- /dev/null +++ b/.github/workflows/update_xdsl_pyodide_build.py @@ -0,0 +1,37 @@ +#!/usr/bin/python3 + +# This script updates the meta.yaml file used by Pyodide to bundle and use xDSL +# Takes the .yaml file and the xDSL directory as arguments + +import yaml +import sys +import os +import hashlib + +meta_yaml_path = sys.argv[1] +xdsl_directory = sys.argv[2] + + +# Parse the auto-generated one +with open(meta_yaml_path) as f: + y = yaml.safe_load(f) + +print(meta_yaml_path) +print(y) + +# Add the frozenlist run-time dependency. Importing in JupyterLite doesn't work properly without it +y["requirements"] = {"run" : ["frozenlist"]} + +# Find the built source distribution. This assumes it is the only thing in xdsl/dist +xdsl_sdist = os.listdir(os.path.join(xdsl_directory, "dist"))[0] +xdsl_sdist = os.path.join(xdsl_directory, "dist", xdsl_sdist) +sha256_hash = hashlib.sha256() +with open(xdsl_sdist,"rb") as sdist: +# Read and update hash string value in blocks of 4K + for byte_block in iter(lambda: sdist.read(4096),b""): + sha256_hash.update(byte_block) + +# Make it build the local xDSL, not the PyPi release. The pyodide build still requires the SHA256 sum. +y["source"] = {"url" : xdsl_sdist, "sha256" : sha256_hash.hexdigest()} +with open(meta_yaml_path, "w") as f: + yaml.dump(y, f) From cea8fef6a6bb566fe309d59d348ea7445dccbbe5 Mon Sep 17 00:00:00 2001 From: Emilien Bauer Date: Fri, 13 Jan 2023 10:35:08 +0000 Subject: [PATCH 4/5] Format CI Python script --- .../workflows/update_xdsl_pyodide_build.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/update_xdsl_pyodide_build.py b/.github/workflows/update_xdsl_pyodide_build.py index 27e3b7084f..141296479d 100755 --- a/.github/workflows/update_xdsl_pyodide_build.py +++ b/.github/workflows/update_xdsl_pyodide_build.py @@ -11,27 +11,26 @@ meta_yaml_path = sys.argv[1] xdsl_directory = sys.argv[2] - # Parse the auto-generated one with open(meta_yaml_path) as f: - y = yaml.safe_load(f) - + y = yaml.safe_load(f) + print(meta_yaml_path) print(y) - + # Add the frozenlist run-time dependency. Importing in JupyterLite doesn't work properly without it -y["requirements"] = {"run" : ["frozenlist"]} +y["requirements"] = {"run": ["frozenlist"]} # Find the built source distribution. This assumes it is the only thing in xdsl/dist xdsl_sdist = os.listdir(os.path.join(xdsl_directory, "dist"))[0] xdsl_sdist = os.path.join(xdsl_directory, "dist", xdsl_sdist) sha256_hash = hashlib.sha256() -with open(xdsl_sdist,"rb") as sdist: -# Read and update hash string value in blocks of 4K - for byte_block in iter(lambda: sdist.read(4096),b""): +with open(xdsl_sdist, "rb") as sdist: + # Read and update hash string value in blocks of 4K + for byte_block in iter(lambda: sdist.read(4096), b""): sha256_hash.update(byte_block) # Make it build the local xDSL, not the PyPi release. The pyodide build still requires the SHA256 sum. -y["source"] = {"url" : xdsl_sdist, "sha256" : sha256_hash.hexdigest()} +y["source"] = {"url": xdsl_sdist, "sha256": sha256_hash.hexdigest()} with open(meta_yaml_path, "w") as f: - yaml.dump(y, f) + yaml.dump(y, f) From e920ca709547ddb5734b412708ddc01a23e95e8a Mon Sep 17 00:00:00 2001 From: Emilien Bauer Date: Fri, 13 Jan 2023 13:20:36 +0000 Subject: [PATCH 5/5] Rename y to yaml_doc. --- .github/workflows/update_xdsl_pyodide_build.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/update_xdsl_pyodide_build.py b/.github/workflows/update_xdsl_pyodide_build.py index 141296479d..4bd9780d15 100755 --- a/.github/workflows/update_xdsl_pyodide_build.py +++ b/.github/workflows/update_xdsl_pyodide_build.py @@ -13,13 +13,10 @@ # Parse the auto-generated one with open(meta_yaml_path) as f: - y = yaml.safe_load(f) - -print(meta_yaml_path) -print(y) + yaml_doc = yaml.safe_load(f) # Add the frozenlist run-time dependency. Importing in JupyterLite doesn't work properly without it -y["requirements"] = {"run": ["frozenlist"]} +yaml_doc["requirements"] = {"run": ["frozenlist"]} # Find the built source distribution. This assumes it is the only thing in xdsl/dist xdsl_sdist = os.listdir(os.path.join(xdsl_directory, "dist"))[0] @@ -31,6 +28,6 @@ sha256_hash.update(byte_block) # Make it build the local xDSL, not the PyPi release. The pyodide build still requires the SHA256 sum. -y["source"] = {"url": xdsl_sdist, "sha256": sha256_hash.hexdigest()} +yaml_doc["source"] = {"url": xdsl_sdist, "sha256": sha256_hash.hexdigest()} with open(meta_yaml_path, "w") as f: - yaml.dump(y, f) + yaml.dump(yaml_doc, f)