diff --git a/.github/workflows/jupyterlite.yml b/.github/workflows/jupyterlite.yml index ee5447b971..9fa2ffc043 100644 --- a/.github/workflows/jupyterlite.yml +++ b/.github/workflows/jupyterlite.yml @@ -11,6 +11,8 @@ jobs: steps: - name: Checkout xDSL uses: actions/checkout@v3 + with: + path: xdsl - name: Setup Python uses: actions/setup-python@v4 @@ -18,61 +20,62 @@ jobs: python-version: '3.10' - name: Install dependencies - run: python -m pip install jupyterlite[all] libarchive-c jupyter_server jupyterlab_server build pkginfo pyodide-build - - - name: Setup EmSDK - # Doing this in a hacky script because the custom Action is requiring an explicit EmSDK version - # Pyodide is requiring a specific version too and it is changing regularly. This install the actual - # version required by the latest pyodide. - - # Double initialization of PYODIDE_EMSCRIPTEN_VERSION: see https://github.com/pyodide/pyodide/issues/3430 run: | - git clone https://github.com/emscripten-core/emsdk.git - cd emsdk - PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) - PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) - ./emsdk install ${PYODIDE_EMSCRIPTEN_VERSION} - ./emsdk activate ${PYODIDE_EMSCRIPTEN_VERSION} - - - name: Build WASM wheels for frozenlist, coverage, and xDSL + python -m pip install jupyterlite[all] libarchive-c build pyodide-build + + - name: Build xDSL source distribution run: | - mkdir pypi - source emsdk/emsdk_env.sh - for i in frozenlist coverage - do + cd xdsl + python setup.py sdist - python -m pip download --no-binary :all: "$(grep $i requirements.txt)" - tar -xf $i-*.tar.gz - rm $i_*.tar.gz - cd $i-* - pyodide build - cd .. - cp $(find -path "./$i-*/dist/$i-*_wasm32.whl") pypi + # Pyodide is cached, so cloned only if not present in the cache, otherwise + # just checked out to whatever desired version and partially rebuilt. - done + - name: Restore cached Pyodide tree + id: cache-pyodide + uses: actions/cache@v3 + with: + path: pyodide + key: pyodide + + - name: Clone pyodide if not cached + if: steps.cache-pyodide.outputs.cache-hit != 'true' + run: git clone https://github.com/pyodide/pyodide.git - pyodide build - cp $(find -path "./dist/xdsl-*.whl") pypi + # Clean the xDSL and FrozenList package folders, generate their skeletons + # and do the necessary updates before building. + - name: Build custom Pyodide distribution + run: | + + cd pyodide + git fetch --all + git checkout 0.22.0a3 + python -m pip install -r requirements.txt + sudo apt update && sudo apt install f2c + + rm packages/xdsl packages/frozenlist -rf + pyodide skeleton pypi xdsl + pyodide skeleton pypi frozenlist + + ../xdsl/.github/workflows/update_xdsl_pyodide_build.py packages/xdsl/meta.yaml ../xdsl + + PYODIDE_PACKAGES="frozenlist,coverage,xdsl" make - name: Build the JupyterLite site run: | mkdir content - cp README.md content - cp docs content -r - for f in $(find -path ./content/*.ipynb) - do - - echo appending to $f - sed -i 's/# xDSL should be available in the environment\./# xDSL should be available in the environment.\\n%pip install xdsl/g' $f + cp xdsl/docs/* content -r - done + rm -rf pyodide/pyodide + mkdir pyodide/pyodide + mv pyodide/dist pyodide/pyodide/pyodide - python -m jupyter lite build --contents content --output-dir jupyterlite + python -m jupyter lite build --contents content --pyodide pyodide/pyodide - name: Upload artifact uses: actions/upload-pages-artifact@v1 with: - path: ./jupyterlite + path: ./_output deploy: needs: build diff --git a/.github/workflows/update_xdsl_pyodide_build.py b/.github/workflows/update_xdsl_pyodide_build.py new file mode 100755 index 0000000000..4bd9780d15 --- /dev/null +++ b/.github/workflows/update_xdsl_pyodide_build.py @@ -0,0 +1,33 @@ +#!/usr/bin/python3 + +# This script updates the meta.yaml file used by Pyodide to bundle and use xDSL +# Takes the .yaml file and the xDSL directory as arguments + +import yaml +import sys +import os +import hashlib + +meta_yaml_path = sys.argv[1] +xdsl_directory = sys.argv[2] + +# Parse the auto-generated one +with open(meta_yaml_path) as f: + yaml_doc = yaml.safe_load(f) + +# Add the frozenlist run-time dependency. Importing in JupyterLite doesn't work properly without it +yaml_doc["requirements"] = {"run": ["frozenlist"]} + +# Find the built source distribution. This assumes it is the only thing in xdsl/dist +xdsl_sdist = os.listdir(os.path.join(xdsl_directory, "dist"))[0] +xdsl_sdist = os.path.join(xdsl_directory, "dist", xdsl_sdist) +sha256_hash = hashlib.sha256() +with open(xdsl_sdist, "rb") as sdist: + # Read and update hash string value in blocks of 4K + for byte_block in iter(lambda: sdist.read(4096), b""): + sha256_hash.update(byte_block) + +# Make it build the local xDSL, not the PyPi release. The pyodide build still requires the SHA256 sum. +yaml_doc["source"] = {"url": xdsl_sdist, "sha256": sha256_hash.hexdigest()} +with open(meta_yaml_path, "w") as f: + yaml.dump(yaml_doc, f) diff --git a/README.md b/README.md index e5b0026ce0..93b212e097 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ The following tutorials will present xDSL basic concepts, how to use its irdl dialect to define new dialects in a user-friendly way, and how to work with both xDSL and MLIR. -- [A simple introduction](https://xdsl.dev/xdsl/retro/notebooks/?path=docs/tutorial.ipynb) -- [A DSL for defining new IRs](https://xdsl.dev/xdsl/retro/notebooks/?path=docs/irdl.ipynb) +- [A simple introduction](https://xdsl.dev/xdsl/retro/notebooks/?path=tutorial.ipynb) +- [A DSL for defining new IRs](https://xdsl.dev/xdsl/retro/notebooks/?path=irdl.ipynb) - [Connecting xDSL with MLIR](docs/mlir_interoperation.md) ## xDSL Developer Setup diff --git a/docs/irdl.ipynb b/docs/irdl.ipynb index 56abc5ca29..7bfd1b2f02 100644 --- a/docs/irdl.ipynb +++ b/docs/irdl.ipynb @@ -26,8 +26,6 @@ "metadata": {}, "outputs": [], "source": [ - "# xDSL should be available in the environment.\n", - "\n", "from xdsl.ir import MLContext\n", "from xdsl.dialects.arith import Arith\n", "from xdsl.dialects.builtin import Builtin\n", diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb index 6eda62fa7f..5a265e14c1 100644 --- a/docs/tutorial.ipynb +++ b/docs/tutorial.ipynb @@ -25,8 +25,6 @@ "metadata": {}, "outputs": [], "source": [ - "# xDSL should be available in the environment.\n", - "\n", "from xdsl.ir import MLContext\n", "from xdsl.dialects.arith import Arith\n", "from xdsl.dialects.func import Func\n",