CI: Build a xDSL-embedding Pyodide distribution.

.github/workflows/jupyterlite.yml

@@ -11,68 +11,71 @@ jobs:
     steps:
       - name: Checkout xDSL
         uses: actions/checkout@v3
+        with:
+          path: xdsl
 
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
 
       - name: Install dependencies
-        run: python -m pip install jupyterlite[all] libarchive-c jupyter_server jupyterlab_server build pkginfo pyodide-build
-
-      - name: Setup EmSDK
-        # Doing this in a hacky script because the custom Action is requiring an explicit EmSDK version
-        # Pyodide is requiring a specific version too and it is changing regularly. This install the actual
-        # version required by the latest pyodide.
-
-        # Double initialization of PYODIDE_EMSCRIPTEN_VERSION: see https://github.com/pyodide/pyodide/issues/3430
         run: |
-          git clone https://github.com/emscripten-core/emsdk.git
-          cd emsdk
-          PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version)
-          PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version)
-          ./emsdk install ${PYODIDE_EMSCRIPTEN_VERSION}
-          ./emsdk activate ${PYODIDE_EMSCRIPTEN_VERSION}
-
-      - name: Build WASM wheels for frozenlist, coverage, and xDSL
+          python -m pip install jupyterlite[all] libarchive-c build pyodide-build
+
+      - name: Build xDSL source distribution
         run: |
-          mkdir pypi
-          source emsdk/emsdk_env.sh
-          for i in frozenlist coverage
-          do
+          cd xdsl
+          python setup.py sdist
 
-          python -m pip download --no-binary :all: "$(grep $i requirements.txt)"
-          tar -xf $i-*.tar.gz
-          rm $i_*.tar.gz
-          cd $i-*
-          pyodide build
-          cd ..
-          cp $(find -path "./$i-*/dist/$i-*_wasm32.whl") pypi
+      # Pyodide is cached, so cloned only if not present in the cache, otherwise
+      # just checked out to whatever desired version and partially rebuilt.
 
-          done
+      - name: Restore cached Pyodide tree
+        id: cache-pyodide
+        uses: actions/cache@v3
+        with:
+          path: pyodide
+          key: pyodide
+
+      - name: Clone pyodide if not cached
+        if: steps.cache-pyodide.outputs.cache-hit != 'true'
+        run: git clone https://github.com/pyodide/pyodide.git
 
-          pyodide build
-          cp $(find -path "./dist/xdsl-*.whl") pypi
+      # Clean the xDSL and FrozenList package folders, generate their skeletons
+      # and do the necessary updates before building.
+      - name: Build custom Pyodide distribution
+        run: |
+
+          cd pyodide
+          git fetch --all
+          git checkout 0.22.0a3
+          python -m pip install -r requirements.txt
+          sudo apt update && sudo apt install f2c
+
+          rm packages/xdsl packages/frozenlist -rf
+          pyodide skeleton pypi xdsl
+          pyodide skeleton pypi frozenlist
+
+          ../xdsl/.github/workflows/update_xdsl_pyodide_build.py packages/xdsl/meta.yaml ../xdsl
+
+          PYODIDE_PACKAGES="frozenlist,coverage,xdsl" make
 
       - name: Build the JupyterLite site
         run: |
           mkdir content
-          cp README.md content
-          cp docs content -r
-          for f in $(find -path ./content/*.ipynb)
-          do
-
-          echo appending to $f
-          sed -i 's/# xDSL should be available in the environment\./# xDSL should be available in the environment.\\n%pip install xdsl/g' $f
+          cp xdsl/docs/* content -r
 
-          done
+          rm -rf pyodide/pyodide
+          mkdir pyodide/pyodide
+          mv pyodide/dist pyodide/pyodide/pyodide
 
-          python -m jupyter lite build --contents content --output-dir jupyterlite
+          python -m jupyter lite build --contents content --pyodide pyodide/pyodide
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v1
         with:
-          path: ./jupyterlite
+          path: ./_output
 
   deploy:
     needs: build

.github/workflows/update_xdsl_pyodide_build.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python3
+
+# This script updates the meta.yaml file used by Pyodide to bundle and use xDSL
+# Takes the .yaml file and the xDSL directory as arguments
+
+import yaml
+import sys
+import os
+import hashlib
+
+meta_yaml_path = sys.argv[1]
+xdsl_directory = sys.argv[2]
+
+# Parse the auto-generated one
+with open(meta_yaml_path) as f:
+    yaml_doc = yaml.safe_load(f)
+
+# Add the frozenlist run-time dependency. Importing in JupyterLite doesn't work properly without it
+yaml_doc["requirements"] = {"run": ["frozenlist"]}
+
+# Find the built source distribution. This assumes it is the only thing in xdsl/dist
+xdsl_sdist = os.listdir(os.path.join(xdsl_directory, "dist"))[0]
+xdsl_sdist = os.path.join(xdsl_directory, "dist", xdsl_sdist)
+sha256_hash = hashlib.sha256()
+with open(xdsl_sdist, "rb") as sdist:
+    # Read and update hash string value in blocks of 4K
+    for byte_block in iter(lambda: sdist.read(4096), b""):
+        sha256_hash.update(byte_block)
+
+# Make it build the local xDSL, not the PyPi release. The pyodide build still requires the SHA256 sum.
+yaml_doc["source"] = {"url": xdsl_sdist, "sha256": sha256_hash.hexdigest()}
+with open(meta_yaml_path, "w") as f:
+    yaml.dump(yaml_doc, f)

README.md

@@ -48,8 +48,8 @@ The following tutorials will present xDSL basic concepts, how to
 use its irdl dialect to define new dialects in a user-friendly way, and how to
 work with both xDSL and MLIR.
 
-- [A simple introduction](https://xdsl.dev/xdsl/retro/notebooks/?path=docs/tutorial.ipynb)
-- [A DSL for defining new IRs](https://xdsl.dev/xdsl/retro/notebooks/?path=docs/irdl.ipynb)
+- [A simple introduction](https://xdsl.dev/xdsl/retro/notebooks/?path=tutorial.ipynb)
+- [A DSL for defining new IRs](https://xdsl.dev/xdsl/retro/notebooks/?path=irdl.ipynb)
 - [Connecting xDSL with MLIR](docs/mlir_interoperation.md)
 
 ## xDSL Developer Setup

docs/irdl.ipynb

@@ -26,8 +26,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# xDSL should be available in the environment.\n",
-    "\n",
     "from xdsl.ir import MLContext\n",
     "from xdsl.dialects.arith import Arith\n",
     "from xdsl.dialects.builtin import Builtin\n",

docs/tutorial.ipynb

@@ -25,8 +25,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# xDSL should be available in the environment.\n",
-    "\n",
     "from xdsl.ir import MLContext\n",
     "from xdsl.dialects.arith import Arith\n",
     "from xdsl.dialects.func import Func\n",